{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "86a4bd36",
   "metadata": {},
   "source": [
    "\n",
    "\n",
    "下面展示基于隐马尔科夫模型的序列标注监督学习的代码。这里以命名实体识别任务为例，所使用的数据是Books数据集。为简单起见，标签序列采用BIO格式。首先构建数据集和标签集合："
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "41673bbb",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'B-NAME': 1, 'I-ORG': 2, 'B-PRO': 3, 'B-EDU': 4, 'I-NAME': 5, 'B-LOC': 6, 'B-TITLE': 7, 'B-RACE': 8, 'I-TITLE': 9, 'I-RACE': 10, 'I-PRO': 11, 'B-CONT': 12, 'I-EDU': 13, 'I-LOC': 14, 'I-CONT': 15, 'B-ORG': 16, 'O': 0}\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import json\n",
    "from collections import defaultdict\n",
    "\n",
    "\n",
    "class NERDataset:\n",
    "    def __init__(self):\n",
    "        train_file, test_file, label_file = 'ner_train.jsonl',\\\n",
    "            'ner_test.jsonl', 'ner_labels.json'\n",
    "        \n",
    "        def read_file(file_name):\n",
    "            with open(file_name, 'r', encoding='utf-8') as fin:\n",
    "                json_list = list(fin)\n",
    "            data_split = []\n",
    "            for json_str in json_list:\n",
    "                raw = json.loads(json_str)\n",
    "                d = {'tokens': raw['tokens'], 'tags': raw['tags']}\n",
    "                data_split.append(d)\n",
    "            return data_split\n",
    "        \n",
    "        # 读取JSON文件，转化为Python对象\n",
    "        self.train_data, self.test_data = read_file(train_file),\\\n",
    "            read_file(test_file)\n",
    "        self.label2id = json.loads(open(label_file, 'r').read())\n",
    "        self.id2label = {}\n",
    "        for k, v in self.label2id.items():\n",
    "            self.id2label[v] = k\n",
    "        print(self.label2id)\n",
    "\n",
    "    # 建立词表，过滤低频词\n",
    "    def build_vocab(self, min_freq=3):\n",
    "        # 统计词频\n",
    "        frequency = defaultdict(int)\n",
    "        for data in self.train_data:\n",
    "            tokens = data['tokens']\n",
    "            for token in tokens:\n",
    "                frequency[token] += 1\n",
    "                \n",
    "        print(f'unique tokens = {len(frequency)}, '+\\\n",
    "              f'total counts = {sum(frequency.values())}, '+\\\n",
    "              f'max freq = {max(frequency.values())}, '+\\\n",
    "              f'min freq = {min(frequency.values())}')\n",
    "        \n",
    "        # 由于词与标签一一对应，不能随便删除字符，\n",
    "        # 因此加入<unk>用于替代未登录词，加入<pad>用于批量训练\n",
    "        self.token2id = {'<pad>': 0, '<unk>': 1}\n",
    "        self.id2token = {0: '<pad>', 1: '<unk>'}\n",
    "        total_count = 0\n",
    "        # 将高频词加入词表\n",
    "        for token, freq in sorted(frequency.items(),\\\n",
    "                key=lambda x: -x[1]):\n",
    "            if freq > min_freq:\n",
    "                self.token2id[token] = len(self.token2id)\n",
    "                self.id2token[len(self.id2token)] = token\n",
    "                total_count += freq\n",
    "            else:\n",
    "                break\n",
    "        print(f'min_freq = {min_freq}, '\n",
    "              f'remaining tokens = {len(self.token2id)}, '\n",
    "              f'in-vocab rate = {total_count / sum(frequency.values())}')\n",
    "\n",
    "    # 将文本输入转化为词表中对应的索引\n",
    "    def convert_tokens_to_ids(self):\n",
    "        for data_split in [self.train_data, self.test_data]:\n",
    "            for data in data_split:\n",
    "                data['token_ids'] = []\n",
    "                for token in data['tokens']:\n",
    "                    data['token_ids'].append(self.token2id.get(token, 1)) \n",
    "        \n",
    "dataset = NERDataset()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "25650a7c",
   "metadata": {},
   "source": [
    "接下来建立词表，将词元转换为索引，并将数据转化为适合训练的格式。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "a3eb49e5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "unique tokens = 2454, total counts = 182068, max freq = 5630, min freq = 1\n",
      "min_freq = 0, remaining tokens = 2456, in-vocab rate = 1.0\n",
      "[595, 510, 353, 263, 424, 225, 764, 637, 353, 65, 80, 47, 41, 74, 53, 41, 141, 23, 74, 7, 259, 27, 135, 47, 31, 74, 67, 25, 1605, 815, 1157, 225, 595, 1158, 738, 688, 353, 1025, 65, 234, 27, 135, 316, 41, 31, 7, 80, 41, 42, 31, 47, 27, 528, 41, 141, 27, 135, 67, 1005, 1606, 363, 19, 677, 294, 37, 678, 1756, 604, 873, 1006, 1005, 353, 33, 30, 9, 113, 5, 45, 11, 82, 60, 21, 26, 5, 186, 153, 100, 565, 873, 353, 424, 1993, 387, 346, 250, 13, 5, 102, 214, 25, 11, 82, 60, 21, 26, 5, 186, 2, 7, 459, 97, 89, 147, 140, 139, 69, 46, 21, 12, 2, 7, 964, 98, 240, 677, 59, 9, 103, 11, 82, 60, 335, 200, 76, 26, 6, 110, 20, 9, 15, 4, 122, 539, 241, 99, 621, 40, 200, 57, 82, 156, 25, 11, 82, 381, 371, 91, 202, 6, 52, 2, 7, 670, 100, 84, 204, 53, 120, 27, 42, 3, 56, 1159, 3, 329, 31, 265, 31, 3, 56, 394, 394, 3, 84, 357, 84, 7, 25, 397, 7, 41, 7, 74, 7, 135, 7, 31, 7, 87, 7, 259, 7, 31, 7, 74, 7, 41, 7, 131, 7, 28, 289, 385, 4]\n",
      "['阿', '里', '斯', '提', '德', '·', '波', '拉', '斯', '（', 'A', 'r', 'i', 's', 't', 'i', 'd', 'e', 's', ' ', 'B', 'o', 'u', 'r', 'a', 's', '）', '和', '卢', '卡', '雅', '·', '阿', '伊', '纳', '罗', '斯', '托', '（', 'L', 'o', 'u', 'k', 'i', 'a', ' ', 'A', 'i', 'n', 'a', 'r', 'o', 'z', 'i', 'd', 'o', 'u', '）', '夫', '妇', '二', '人', '均', '拥', '有', '希', '腊', '比', '雷', '埃', '夫', '斯', '技', '术', '教', '育', '学', '院', '计', '算', '机', '工', '程', '学', '位', '以', '及', '色', '雷', '斯', '德', '谟', '克', '利', '特', '大', '学', '电', '子', '和', '计', '算', '机', '工', '程', '学', '位', '，', ' ', '都', '从', '事', '过', '软', '件', '开', '发', '工', '作', '，', ' ', '且', '目', '前', '均', '为', '教', '授', '计', '算', '机', '相', '关', '课', '程', '的', '高', '中', '教', '师', '。', '他', '们', '写', '了', '很', '多', '关', '于', '算', '法', '和', '计', '算', '思', '维', '方', '面', '的', '书', '，', ' ', '涉', '及', 'P', 'y', 't', 'h', 'o', 'n', '、', 'C', '#', '、', 'J', 'a', 'v', 'a', '、', 'C', '+', '+', '、', 'P', 'H', 'P', ' ', '和', 'V', ' ', 'i', ' ', 's', ' ', 'u', ' ', 'a', ' ', 'l', ' ', 'B', ' ', 'a', ' ', 's', ' ', 'i', ' ', 'c', ' ', '等', '语', '言', '。']\n"
     ]
    }
   ],
   "source": [
    "# 截取一部分数据便于更快训练\n",
    "dataset.train_data = dataset.train_data[:1000]\n",
    "dataset.test_data = dataset.test_data[:200]\n",
    "\n",
    "dataset.build_vocab(min_freq=0)\n",
    "dataset.convert_tokens_to_ids()\n",
    "print(dataset.train_data[0]['token_ids'])\n",
    "print([dataset.id2token[token_id] for token_id in \\\n",
    "       dataset.train_data[0]['token_ids']])\n",
    "\n",
    "def collect_data(data_split):\n",
    "    X, Y = [], []\n",
    "    for data in data_split:\n",
    "        assert len(data['token_ids']) == len(data['tags'])\n",
    "        X.append(data['token_ids'])\n",
    "        Y.append(data['tags'])\n",
    "    return X, Y\n",
    "\n",
    "train_X, train_Y = collect_data(dataset.train_data)\n",
    "test_X, test_Y = collect_data(dataset.test_data)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7de17192",
   "metadata": {},
   "source": [
    "随后实现隐马尔科夫模型，使用最大似然估计得到模型参数，使用维特比算法进行解码。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "3229fd6a",
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "class HMM:\n",
    "    def __init__(self, n_tags, n_tokens):\n",
    "        self.n_tags = n_tags\n",
    "        self.n_tokens = n_tokens\n",
    "    \n",
    "    # 使用最大似然估计计算模型参数\n",
    "    def fit(self, X, Y):\n",
    "        Y0_cnt = np.zeros(self.n_tags)\n",
    "        YY_cnt = np.zeros((self.n_tags, self.n_tags))\n",
    "        YX_cnt = np.zeros((self.n_tags, self.n_tokens))\n",
    "        for x, y in zip(X, Y):\n",
    "            Y0_cnt[y[0]] += 1\n",
    "            last_y = y[0]\n",
    "            for i in range(1, len(y)):\n",
    "                YY_cnt[last_y, y[i]] += 1\n",
    "                last_y = y[i]\n",
    "            for xi, yi in zip(x, y):\n",
    "                YX_cnt[yi, xi] += 1\n",
    "        self.init_prob = Y0_cnt / Y0_cnt.sum()\n",
    "        self.transition_prob = YY_cnt\n",
    "        self.emission_prob = YX_cnt\n",
    "        for i in range(self.n_tags):\n",
    "            # 为了避免训练集过小时除0\n",
    "            yy_sum = YY_cnt[i].sum()\n",
    "            if yy_sum > 0:\n",
    "                self.transition_prob[i] = YY_cnt[i] / yy_sum\n",
    "            yx_sum = YX_cnt[i].sum()\n",
    "            if yx_sum > 0:\n",
    "                self.emission_prob[i] = YX_cnt[i] / yx_sum\n",
    "    \n",
    "    # 已知模型参数的条件下，使用维特比算法解码得到最优标签序列\n",
    "    def viterbi(self, x):\n",
    "        assert hasattr(self, 'init_prob') and hasattr(self,\\\n",
    "            'transition_prob') and hasattr(self, 'emission_prob')\n",
    "        Pi = np.zeros((len(x), self.n_tags))\n",
    "        Y = np.zeros((len(x), self.n_tags), dtype=np.int32)\n",
    "        # 初始化\n",
    "        for i in range(self.n_tags):\n",
    "            Pi[0, i] = self.init_prob[i] * self.emission_prob[i, x[0]]\n",
    "            Y[0, i] = -1\n",
    "        for t in range(1, len(x)):\n",
    "            for i in range(self.n_tags):\n",
    "                tmp = []\n",
    "                for j in range(self.n_tags):\n",
    "                    tmp.append(self.transition_prob[j, i] * Pi[t-1, j])\n",
    "                best_j = np.argmax(tmp)\n",
    "                # 维特比算法递推公式\n",
    "                Pi[t, i] = self.emission_prob[i, x[t]] * tmp[best_j]\n",
    "                Y[t, i] = best_j\n",
    "        y = [np.argmax(Pi[-1])]\n",
    "        for t in range(len(x)-1, 0, -1):\n",
    "            y.append(Y[t, y[-1]])\n",
    "        return np.max(Pi[len(x)-1]), y[::-1]\n",
    "    \n",
    "    def decode(self, X):\n",
    "        Y = []\n",
    "        for x in X:\n",
    "            _, y = self.viterbi(x)\n",
    "            Y.append(y)\n",
    "        return Y\n",
    "\n",
    "hmm = HMM(len(dataset.label2id), len(dataset.token2id))\n",
    "hmm.fit(train_X, train_Y)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "32868ed6",
   "metadata": {},
   "source": [
    "最后验证模型效果"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "4f4b7e88",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "precision = 0.45762149610217284, recall = 0.3289222699093944, f1 = 0.38274259554692375\n",
      "precision = 0.4189636163175303, recall = 0.23270055113288426, f1 = 0.2992125984251969\n"
     ]
    }
   ],
   "source": [
    "def extract_entity(labels):\n",
    "    entity_list = []\n",
    "    entity_start = -1\n",
    "    entity_length = 0\n",
    "    entity_type = None\n",
    "    for token_index, label in enumerate(labels):\n",
    "        if label.startswith('B'):\n",
    "            if entity_start != -1:\n",
    "                # 遇到了一个新的B，将上一个实体加入列表\n",
    "                entity_list.append((entity_start, entity_length,\\\n",
    "                    entity_type))\n",
    "            # 记录新实体\n",
    "            entity_start = token_index\n",
    "            entity_length = 1\n",
    "            entity_type = label.split('-')[1]\n",
    "        elif label.startswith('I'):\n",
    "            if entity_start != -1:\n",
    "                # 上一个实体未关闭，遇到了一个新的I\n",
    "                if entity_type == label.split('-')[1]:\n",
    "                    # 若上一个实体与当前类型相同，长度+1\n",
    "                    entity_length += 1\n",
    "                else:\n",
    "                    # 若上一个实体与当前类型不同，\n",
    "                    # 将上一个实体加入列表，重置实体\n",
    "                    entity_list.append((entity_start, entity_length,\\\n",
    "                        entity_type))\n",
    "                    entity_start = -1\n",
    "                    entity_length = 0\n",
    "                    entity_type = None\n",
    "        else:\n",
    "            if entity_start != -1:\n",
    "                # 遇到了一个新的O，将上一个实体加入列表\n",
    "                entity_list.append((entity_start, entity_length,\\\n",
    "                    entity_type))\n",
    "            # 重置实体\n",
    "            entity_start = -1\n",
    "            entity_length = 0\n",
    "            entity_type = None\n",
    "    if entity_start != -1:\n",
    "        # 将上一个实体加入列表\n",
    "        entity_list.append((entity_start, entity_length, entity_type))\n",
    "    return entity_list\n",
    "\n",
    "def compute_metric(Y, P):\n",
    "    true_entity_set = set()\n",
    "    pred_entity_set = set()\n",
    "    for sent_no, labels in enumerate(Y):\n",
    "        labels = [dataset.id2label[x] for x in labels]\n",
    "        for ent in extract_entity(labels):\n",
    "            true_entity_set.add((sent_no, ent))\n",
    "    for sent_no, labels in enumerate(P):\n",
    "        labels = [dataset.id2label[x] for x in labels]\n",
    "        for ent in extract_entity(labels):\n",
    "            pred_entity_set.add((sent_no, ent))\n",
    "    if len(true_entity_set) > 0:\n",
    "        recall = len(true_entity_set & pred_entity_set)\\\n",
    "            / len(true_entity_set)\n",
    "    else:\n",
    "        recall = 0\n",
    "    if len(pred_entity_set) > 0:\n",
    "        precision = len(true_entity_set & pred_entity_set)\\\n",
    "            / len(pred_entity_set)\n",
    "    else:\n",
    "        precision = 0\n",
    "    if precision > 0 and recall > 0:\n",
    "        f1 = 2 * precision * recall / (precision + recall)\n",
    "    else:\n",
    "        f1 = 0\n",
    "    return precision, recall, f1\n",
    "\n",
    "train_P = hmm.decode(train_X)\n",
    "p, r, f = compute_metric(train_Y, train_P)\n",
    "print(f'precision = {p}, recall = {r}, f1 = {f}')\n",
    "test_P = hmm.decode(test_X)\n",
    "p, r, f = compute_metric(test_Y, test_P)\n",
    "print(f'precision = {p}, recall = {r}, f1 = {f}')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3abe8397",
   "metadata": {},
   "source": [
    "\n",
    "\n",
    "下面展示基于条件随机场的序列标注模型。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "787f8ad1",
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "代码修改自GitHub项目kmkurn/pytorch-crf\n",
    "（Copyright (c) 2019, Kemal Kurniawan, MIT License（见附录））\n",
    "\"\"\"\n",
    "import torch\n",
    "from torch import nn\n",
    "\n",
    "class CRFLayer(nn.Module):\n",
    "    def __init__(self, n_tags, n_features):\n",
    "        super().__init__()\n",
    "        self.n_tags = n_tags\n",
    "        self.n_features = n_features\n",
    "        # 定义模型参数\n",
    "        self.transitions = nn.Parameter(torch.empty(\\\n",
    "            n_tags, n_tags))\n",
    "        self.emission_weight = nn.Parameter(torch.empty(\\\n",
    "            n_features, n_tags))\n",
    "        self.start_transitions = nn.Parameter(torch.empty(n_tags))\n",
    "        self.end_transitions = nn.Parameter(torch.empty(n_tags))\n",
    "        self.reset_parameters()\n",
    "        \n",
    "    # 使用（-0.1,0.1）之间的均匀分布初始化参数\n",
    "    def reset_parameters(self):\n",
    "        nn.init.uniform_(self.transitions, -0.1, 0.1)\n",
    "        nn.init.uniform_(self.emission_weight, -0.1, 0.1)\n",
    "        nn.init.uniform_(self.start_transitions, -0.1, 0.1)\n",
    "        nn.init.uniform_(self.end_transitions, -0.1, 0.1)\n",
    "    \n",
    "    # 使用动态规划计算得分\n",
    "    def compute_score(self, emissions, tags, masks):\n",
    "        seq_len, batch_size, n_tags = emissions.shape\n",
    "        score = self.start_transitions[tags[0]] + \\\n",
    "            emissions[0, torch.arange(batch_size), tags[0]]\n",
    "        \n",
    "        for i in range(1, seq_len):\n",
    "            score += self.transitions[tags[i-1], tags[i]] * masks[i]\n",
    "            score += emissions[i, torch.arange(batch_size),\\\n",
    "                tags[i]] * masks[i]\n",
    "        \n",
    "        seq_ends = masks.long().sum(dim=0) - 1\n",
    "        last_tags = tags[seq_ends, torch.arange(batch_size)]\n",
    "        score += self.end_transitions[last_tags]\n",
    "        return score\n",
    "    \n",
    "    # 计算配分函数\n",
    "    def computer_normalizer(self, emissions, masks):\n",
    "        seq_len, batch_size, n_tags = emissions.shape\n",
    "        # batch_size * n_tags, [起始分数 + y_0为某标签的发射分数 ...]\n",
    "        score = self.start_transitions + emissions[0]\n",
    "        \n",
    "        for i in range(1, seq_len):\n",
    "            # batch_size * n_tags * 1 [y_{i-1}为某tag的总分]\n",
    "            broadcast_score = score.unsqueeze(2)\n",
    "            # batch_size * 1 * n_tags [y_i为某标签的发射分数]\n",
    "            broadcast_emissions = emissions[i].unsqueeze(1)\n",
    "            # batch_size * n_tags * n_tags [任意y_{i-1}到y_i的总分]\n",
    "            next_score = broadcast_score + self.transitions +\\\n",
    "                broadcast_emissions\n",
    "            # batch_size * n_tags [对y_{i-1}求和]\n",
    "            next_score = torch.logsumexp(next_score, dim=1)\n",
    "            # masks为True则更新，否则保留\n",
    "            score = torch.where(masks[i].unsqueeze(1), next_score, score)\n",
    "            \n",
    "        score += self.end_transitions\n",
    "        return torch.logsumexp(score, dim=1)\n",
    "    \n",
    "    def forward(self, features, tags, masks):\n",
    "        \"\"\"\n",
    "        features: seq_len * batch_size * n_features\n",
    "        tags/masks: seq_len * batch_size\n",
    "        \"\"\"\n",
    "        _, batch_size, _ = features.size()\n",
    "        emissions = torch.matmul(features, self.emission_weight)\n",
    "        masks = masks.to(torch.bool)\n",
    "        \n",
    "        score = self.compute_score(emissions, tags, masks)\n",
    "        partition = self.computer_normalizer(emissions, masks)\n",
    "        \n",
    "        likelihood = score - partition\n",
    "        return likelihood.sum() / batch_size\n",
    "    \n",
    "    def decode(self, features, masks):\n",
    "        # 与computer_normalizer类似，sum变为max\n",
    "        emissions = torch.matmul(features, self.emission_weight)\n",
    "        masks = masks.to(torch.bool)\n",
    "        \n",
    "        seq_len, batch_size, n_tags = emissions.shape\n",
    "        score = self.start_transitions + emissions[0]\n",
    "        history = []\n",
    "        \n",
    "        for i in range(1, seq_len):\n",
    "            broadcast_score = score.unsqueeze(2)\n",
    "            broadcast_emission = emissions[i].unsqueeze(1)\n",
    "            \n",
    "            next_score = broadcast_score + self.transitions +\\\n",
    "                broadcast_emission\n",
    "            next_score, indices = next_score.max(dim=1)\n",
    "            \n",
    "            score = torch.where(masks[i].unsqueeze(1), next_score,\\\n",
    "                score)\n",
    "            history.append(indices)\n",
    "            \n",
    "        score += self.end_transitions\n",
    "        seq_ends = masks.long().sum(dim=0) - 1\n",
    "        best_tags_list = []\n",
    "        \n",
    "        for idx in range(batch_size):\n",
    "            _, best_last_tag = score[idx].max(dim=0)\n",
    "            best_tags = [best_last_tag.item()]\n",
    "            \n",
    "            for hist in reversed(history[:seq_ends[idx]]):\n",
    "                best_last_tag = hist[idx][best_tags[-1]]\n",
    "                best_tags.append(best_last_tag.item())\n",
    "                \n",
    "            best_tags.reverse()\n",
    "            best_tags_list.append(best_tags)\n",
    "            \n",
    "        return best_tags_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "69f4af96",
   "metadata": {},
   "outputs": [],
   "source": [
    "class CRF(nn.Module):\n",
    "    def __init__(self, vocab_size, hidden_size, n_tags):\n",
    "        super().__init__()\n",
    "        self.embedding = nn.Embedding(vocab_size, hidden_size)\n",
    "        self.crf = CRFLayer(n_tags, hidden_size)\n",
    "        \n",
    "    def forward(self, input_ids, masks, labels):\n",
    "        \"\"\"\n",
    "        input_ids/masks/labels: batch_size * seq_len\n",
    "        \"\"\"\n",
    "        # 将输入序列转化为词嵌入序列\n",
    "        # batch_size * seq_len * embed_size\n",
    "        embed = self.embedding(input_ids)\n",
    "        embed = torch.transpose(embed, 0, 1)\n",
    "        masks = torch.transpose(masks, 0, 1)\n",
    "        labels = torch.transpose(labels, 0, 1)\n",
    "        # 隐状态和标签、掩码输入条件随机场计算损失\n",
    "        llh = self.crf(embed, labels, masks)\n",
    "        return -llh\n",
    "    \n",
    "    def decode(self, input_ids, masks):\n",
    "        embed = self.embedding(input_ids)\n",
    "        embed = torch.transpose(embed, 0, 1)\n",
    "        masks = torch.transpose(masks, 0, 1)\n",
    "        # 调用CRFLayer进行解码\n",
    "        return self.crf.decode(embed, masks)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "178948b4",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "epoch-19, step=7, loss=54.49: 100%|█████████████| 20/20 [04:53<00:00, 14.68s/it]\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAkQAAAGwCAYAAABIC3rIAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8pXeV/AAAACXBIWXMAAA9hAAAPYQGoP6dpAABXwElEQVR4nO3dd3hUVf4G8PdmWpJJMmkkw0CAAKEmFBOkKrCEItJERQEpv8VOMXRZdEVcieIKKiiuZcEFXVgVEBsCCpFIDwTpNaRAQijJpM9MZu7vj5iLQwohmWTa+3meeR7mzpk735OsmXfPPedcQRRFEURERERuzMPeBRARERHZGwMRERERuT0GIiIiInJ7DERERETk9hiIiIiIyO0xEBEREZHbYyAiIiIitye3dwHOwmKx4MqVK/D19YUgCPYuh4iIiGpAFEXk5+dDp9PBw6PqcSAGohq6cuUKwsLC7F0GERER1UJ6ejqaNm1a5esMRDXk6+sLoOwH6ufnZ+dqiIiIqCby8vIQFhYmfY9XhYGohsovk/n5+TEQEREROZk7TXfhpGoiIiJyewxERERE5PYYiIiIiMjtcQ4RERERALPZDJPJZO8y6C4pFArIZLI6n4eBiIiI3JooisjKykJubq69S6Fa8vf3h1arrdM+gQxERETk1srDUEhICLy9vbn5rhMRRRFFRUXIzs4GADRu3LjW52IgIiIit2U2m6UwFBQUZO9yqBa8vLwAANnZ2QgJCan15TO7Tqr+9ddfMXz4cOh0OgiCgM2bN0uvmUwmzJ8/H1FRUVCr1dDpdJg4cSKuXLlidQ6DwYDp06cjODgYarUaI0aMQEZGhlWbnJwcTJgwARqNBhqNBhMmTODQKBERSXOGvL297VwJ1UX5768uc8DsGogKCwvRuXNnrFy5ssJrRUVFOHz4MF5++WUcPnwYGzduxNmzZzFixAirdnFxcdi0aRPWr1+PxMREFBQUYNiwYTCbzVKbcePGITk5GVu3bsXWrVuRnJyMCRMm1Hv/iIjIOfAymXOzxe9PEEVRtEEtdSYIAjZt2oRRo0ZV2ebgwYO49957kZqaimbNmkGv16NRo0ZYu3YtHnvsMQC37jn2ww8/YPDgwTh16hQ6dOiAffv2oXv37gCAffv2oWfPnjh9+jTatm1bo/ry8vKg0Wig1+u5UzURkYsoKSlBSkoKwsPD4enpae9yqJaq+z3W9PvbqfYh0uv1EAQB/v7+AICkpCSYTCYMGjRIaqPT6RAZGYk9e/YAAPbu3QuNRiOFIQDo0aMHNBqN1KYyBoMBeXl5Vg8iIiJyTU4TiEpKSvDiiy9i3LhxUsLLysqCUqlEQECAVdvQ0FBkZWVJbUJCQiqcLyQkRGpTmfj4eGnOkUaj4Z3uiYjIpbVo0QLvvPOO3c9hL04RiEwmEx5//HFYLBZ88MEHd2wviqLV9cTKri3e3uZ2CxYsgF6vlx7p6em1K/4OTGYLzl7NR34JNwMjIqKa69evH+Li4mx2voMHD+Lpp5+22fmcjcMHIpPJhDFjxiAlJQXbt2+3uv6n1WphNBqRk5Nj9Z7s7GyEhoZKba5evVrhvNeuXZPaVEalUkl3tq/PO9w/vGoPBi3/FXsv3KiX8xMRkfsSRRGlpaU1atuoUSO3Xm3n0IGoPAydO3cOO3bsqLBHRHR0NBQKBbZv3y4dy8zMxPHjx9GrVy8AQM+ePaHX63HgwAGpzf79+6HX66U29tSqkQ8A4OzVfDtXQkREwB+b/RlL7fKo6TqnyZMnIyEhAe+++y4EQYAgCLh06RJ27doFQRDw008/ISYmBiqVCrt378aFCxcwcuRIhIaGwsfHB926dcOOHTusznn75S5BEPDJJ5/goYcegre3NyIiIrBly5a7+lmmpaVh5MiR8PHxgZ+fH8aMGWM1SHH06FH0798fvr6+8PPzQ3R0NA4dOgQASE1NxfDhwxEQEAC1Wo2OHTvihx9+uKvPvxt23ZixoKAA58+fl56npKQgOTkZgYGB0Ol0eOSRR3D48GF89913MJvN0pyfwMBAKJVKaDQaTJkyBbNnz0ZQUBACAwMxZ84cREVFITY2FgDQvn17DBkyBE899RT+9a9/AQCefvppDBs2rMYrzOpTm1BfAMDZqwV2roSIiACg2GRGh7//ZJfPPrl4MLyVd/5qfvfdd3H27FlERkZi8eLFAMpGeC5dugQAmDdvHv75z3+iZcuW8Pf3R0ZGBoYOHYp//OMf8PT0xGeffYbhw4fjzJkzaNasWZWf8+qrr2Lp0qV46623sGLFCowfPx6pqakIDAy8Y42iKGLUqFFQq9VISEhAaWkpnn/+eTz22GPYtWsXAGD8+PHo2rUrVq1aBZlMhuTkZCgUCgDA1KlTYTQa8euvv0KtVuPkyZPw8fG54+fWll0D0aFDh9C/f3/p+axZswAAkyZNwqJFi6Qk2qVLF6v37dy5E/369QMALF++HHK5HGPGjEFxcTEGDBiANWvWWO1U+fnnn2PGjBnSarQRI0ZUuveRPbQJ5QgRERHdHY1GA6VSCW9vb2i12gqvL168GAMHDpSeBwUFoXPnztLzf/zjH9i0aRO2bNmCadOmVfk5kydPxtixYwEAS5YswYoVK3DgwAEMGTLkjjXu2LEDv//+O1JSUqSFSWvXrkXHjh1x8OBBdOvWDWlpaZg7dy7atWsHAIiIiJDen5aWhocffhhRUVEAgJYtW97xM+vCroGoX79+1Q4P1mTo0NPTEytWrMCKFSuqbBMYGIh169bVqsb6Vj5CdOFaAUxmCxQyh76KSUTk8rwUMpxcPNhun20LMTExVs8LCwvx6quv4rvvvsOVK1dQWlqK4uJipKWlVXueTp06Sf9Wq9Xw9fWV7ht2J6dOnUJYWJjVKu0OHTrA398fp06dQrdu3TBr1iw8+eSTWLt2LWJjY/Hoo4+iVatWAIAZM2bgueeew7Zt2xAbG4uHH37Yqh5b47evnTXx94JaKYPJLCL1RqG9yyEicnuCIMBbKbfLw1Y7ZqvVaqvnc+fOxddff43XX38du3fvRnJyMqKiomA0Gqs9T/nlqz//bCwWS41qqGo195+PL1q0CCdOnMCDDz6IX375BR06dMCmTZsAAE8++SQuXryICRMm4NixY4iJial28KOuGIjszMNDQMQfo0RnsjiPiIiIakapVFrdpqo6u3fvxuTJk/HQQw8hKioKWq1Wmm9UXzp06IC0tDSrbWtOnjwJvV6P9u3bS8fatGmDmTNnYtu2bRg9ejRWr14tvRYWFoZnn30WGzduxOzZs/Hxxx/XW70MRA6gfB7RGc4jIiKiGmrRogX279+PS5cu4fr169WO3LRu3RobN25EcnIyjh49inHjxtV4pKe2YmNj0alTJ4wfPx6HDx/GgQMHMHHiRPTt2xcxMTEoLi7GtGnTsGvXLqSmpuK3337DwYMHpbAUFxeHn376CSkpKTh8+DB++eUXqyBlawxEDkBaaZbFQERERDUzZ84cyGQydOjQAY0aNap2PtDy5csREBCAXr16Yfjw4Rg8eDDuueeeeq1PEARs3rwZAQEBuP/++xEbG4uWLVtiw4YNAACZTIYbN25g4sSJaNOmDcaMGYMHHngAr776KgDAbDZj6tSp0mrxtm3b1mhz5lrX6yg3d3V09Xlz193nrmHCpwfQspEav8zuZ9NzExFR1XhzV9fgdjd3dVVt/xghunS9ECWmml0PJiIiItthIHIAjXxV0HgpYBHLlt8TERFRw2IgcgCCIEijROe4YzUREVGDYyByEG20XGlGRGQvnE7r3Gzx+2MgchBtudKMiKjBlW88WFRUZOdKqC7Kf3+3byR5N+x66w66RdqckSNEREQNRiaTwd/fX7odhbe3t812i6b6J4oiioqKkJ2dDX9/f6v7mN4tBiIHUb4XUUZOMQoNpVCr+KshImoI5TdHrek9usjx+Pv7V3qT27vBb10HEahWopGvCtfyDTiXXYAuYf72LomIyC0IgoDGjRsjJCQEJpPJ3uXQXVIoFHUaGSrHQORAIkJ8cC3fgAsMREREDU4mk9nki5WcEydVO5CmAV4AgCu5xXauhIiIyL0wEDkQnf8fgUjPQERERNSQGIgcSHkgupxbYudKiIiI3AsDkQNpWh6IcrgfBhERUUNiIHIg0iWz3BLumkpERNSAGIgciFbjCQAoNpmRW8Sln0RERA2FgciBeCpkCPZRAQAuc6UZERFRg2EgcjBN/MtGibj0noiIqOEwEDmYW/OIGIiIiIgaCgORg7m1FxGX3hMRETUUBiIHc2svIo4QERERNRQGIgfDOUREREQNj4HIwUgjRDkMRERERA2FgcjBNPkjEGXnG2AoNdu5GiIiIvfAQORgAtVKqORlv5areoOdqyEiInIPDEQORhAEaZSIE6uJiIgaBgORA+JeRERERA2LgcgB6bjSjIiIqEExEDmgW5szMhARERE1BAYiB3Rrc0buVk1ERNQQGIgcUBPOISIiImpQDEQO6M+TqkVRtHM1REREro+ByAE11pRNqi4ympFbZLJzNURERK6PgcgBeSpkCPZRAeBeRERERA2BgchB8SavREREDYeByEFxc0YiIqKGw0DkoG7tRcSl90RERPWNgchB6Xg/MyIiogbDQOSgOIeIiIio4TAQOSjOISIiImo4DEQOqjwQZecbYCy12LkaIiIi18ZA5KCC1Eoo5R4QReBqHidWExER1ScGIgclCIJ0T7OMHF42IyIiqk8MRA6MN3klIiJqGHYNRL/++iuGDx8OnU4HQRCwefNmq9dFUcSiRYug0+ng5eWFfv364cSJE1ZtDAYDpk+fjuDgYKjVaowYMQIZGRlWbXJycjBhwgRoNBpoNBpMmDABubm59dy7utNxpRkREVGDsGsgKiwsROfOnbFy5cpKX1+6dCmWLVuGlStX4uDBg9BqtRg4cCDy8/OlNnFxcdi0aRPWr1+PxMREFBQUYNiwYTCbzVKbcePGITk5GVu3bsXWrVuRnJyMCRMm1Hv/6urW5owMRERERPVKdBAAxE2bNknPLRaLqNVqxTfeeEM6VlJSImo0GvHDDz8URVEUc3NzRYVCIa5fv15qc/nyZdHDw0PcunWrKIqiePLkSRGAuG/fPqnN3r17RQDi6dOna1yfXq8XAYh6vb62XbxrGw6mic3nfydO+HR/g30mERGRK6np97fDziFKSUlBVlYWBg0aJB1TqVTo27cv9uzZAwBISkqCyWSyaqPT6RAZGSm12bt3LzQaDbp37y616dGjBzQajdSmMgaDAXl5eVaPhsY5RERERA3DYQNRVlYWACA0NNTqeGhoqPRaVlYWlEolAgICqm0TEhJS4fwhISFSm8rEx8dLc440Gg3CwsLq1J/a+PPmjKIoNvjnExERuQuHDUTlBEGwei6KYoVjt7u9TWXt73SeBQsWQK/XS4/09PS7rLzuGmvKJlUXGc3QF5sa/POJiIjchcMGIq1WCwAVRnGys7OlUSOtVguj0YicnJxq21y9erXC+a9du1Zh9OnPVCoV/Pz8rB4NzVMhQ7CPEgBv8kpERFSfHDYQhYeHQ6vVYvv27dIxo9GIhIQE9OrVCwAQHR0NhUJh1SYzMxPHjx+X2vTs2RN6vR4HDhyQ2uzfvx96vV5q48huXTbjbtVERET1RW7PDy8oKMD58+el5ykpKUhOTkZgYCCaNWuGuLg4LFmyBBEREYiIiMCSJUvg7e2NcePGAQA0Gg2mTJmC2bNnIygoCIGBgZgzZw6ioqIQGxsLAGjfvj2GDBmCp556Cv/6178AAE8//TSGDRuGtm3bNnyn75JO44XfM/S4nFNk71KIiIhcll0D0aFDh9C/f3/p+axZswAAkyZNwpo1azBv3jwUFxfj+eefR05ODrp3745t27bB19dXes/y5cshl8sxZswYFBcXY8CAAVizZg1kMpnU5vPPP8eMGTOk1WgjRoyocu8jR9MkoHwvIo4QERER1RdB5PKlGsnLy4NGo4Fer2/Q+USfJqbgte9O4sFOjfH+uHsa7HOJiIhcQU2/vx12DhGVacLbdxAREdU7BiIHp+PmjERERPWOgcjBlQei7HwDjKUWO1dDRETkmhiIHFyQWgml3AOiCFzN48RqIiKi+sBA5OAEQZDuacbNGYmIiOoHA5ET0HFiNRERUb1iIHICOg0nVhMREdUnBiInoOMlMyIionrFQOQEynervsz7mREREdULBiIn0IR7EREREdUrBiIn8OfNGXmnFSIiIttjIHICjTVlq8yKjGboi012roaIiMj1MBA5AU+FDME+SgCcWE1ERFQfGIicxK3LZpxYTUREZGsMRE6CexERERHVHwYiJ8G73hMREdUfBiInUX77Ds4hIiIisj0GIifBG7wSERHVHwYiJ1G+WzUvmREREdkeA5GTKJ9DlJ1vgLHUYudqiIiIXAsDkZMIUiuhlHtAFIGreVx6T0REZEsMRE5CEATOIyIiIqonDEROpHylGecRERER2RYDkRPh5oxERET1g4HIieikS2acQ0RERGRLDEROpAl3qyYiIqoXDEROhLfvICIiqh8MRE7kz7fvEEXRztUQERG5DgYiJ1I+QlRkNENfbLJzNURERK6DgciJeCpkCPZRAuBeRERERLbEQORkbs0j4kozIiIiW2EgcjJav7J5RFl6jhARERHZCgORk2msKQtEmXqOEBEREdkKA5GT0f6xW3UWAxEREZHNMBA5GY4QERER2R4DkZPR/hGIsvIYiIiIiGyFgcjJ3Boh4uaMREREtsJA5GRC/1hlVmKycHNGIiIiG2EgcjKeChkC1WWbM3IeERERkW0wEDkhaS8iziMiIiKyCQYiJ1Q+j4hL74mIiGyDgcgJabn0noiIyKYYiJzQrREi3r6DiIjIFhiInFD5btUcISIiIrINBiInxDlEREREtsVA5IS0DEREREQ2xUDkhMqX3ecbSpFfws0ZiYiI6oqByAmpVXL4ecoBAFe5FxEREVGdOXQgKi0txUsvvYTw8HB4eXmhZcuWWLx4MSwWi9RGFEUsWrQIOp0OXl5e6NevH06cOGF1HoPBgOnTpyM4OBhqtRojRoxARkZGQ3fHphpzYjUREZHNOHQgevPNN/Hhhx9i5cqVOHXqFJYuXYq33noLK1askNosXboUy5Ytw8qVK3Hw4EFotVoMHDgQ+fn5Upu4uDhs2rQJ69evR2JiIgoKCjBs2DCYzWZ7dMsmuBcRERGR7cjtXUB19u7di5EjR+LBBx8EALRo0QL//e9/cejQIQBlo0PvvPMOFi5ciNGjRwMAPvvsM4SGhuKLL77AM888A71ej08//RRr165FbGwsAGDdunUICwvDjh07MHjw4Eo/22AwwGAwSM/z8vLqs6t3jSvNiIiIbMehR4j69OmDn3/+GWfPngUAHD16FImJiRg6dCgAICUlBVlZWRg0aJD0HpVKhb59+2LPnj0AgKSkJJhMJqs2Op0OkZGRUpvKxMfHQ6PRSI+wsLD66GKtcYSIiIjIdhx6hGj+/PnQ6/Vo164dZDIZzGYzXn/9dYwdOxYAkJWVBQAIDQ21el9oaChSU1OlNkqlEgEBARXalL+/MgsWLMCsWbOk53l5eQ4VirhbNRERke04dCDasGED1q1bhy+++AIdO3ZEcnIy4uLioNPpMGnSJKmdIAhW7xNFscKx292pjUqlgkqlqlsH6hF3qyYiIrIdhw5Ec+fOxYsvvojHH38cABAVFYXU1FTEx8dj0qRJ0Gq1AMpGgRo3biy9Lzs7Wxo10mq1MBqNyMnJsRolys7ORq9evRqwN7YljRBx2T0REVGdOfQcoqKiInh4WJcok8mkZffh4eHQarXYvn279LrRaERCQoIUdqKjo6FQKKzaZGZm4vjx404diEL/2Jwxt8iEYqPzrpYjIiJyBA49QjR8+HC8/vrraNasGTp27IgjR45g2bJl+Otf/wqg7FJZXFwclixZgoiICERERGDJkiXw9vbGuHHjAAAajQZTpkzB7NmzERQUhMDAQMyZMwdRUVHSqjNn5Ocph7dShiKjGVl5JQgPVtu7JCIiIqfl0IFoxYoVePnll/H8888jOzsbOp0OzzzzDP7+979LbebNm4fi4mI8//zzyMnJQffu3bFt2zb4+vpKbZYvXw65XI4xY8aguLgYAwYMwJo1ayCTyezRLZsQBAFajScuXitEpr6YgYiIiKgOBFEURXsX4Qzy8vKg0Wig1+vh5+dn73IAAOM/2Yffzt/AsjGdMfqepvYuh4iIyOHU9PvboecQUfW0flxpRkREZAsMRE6Mu1UTERHZBgORE+Nu1URERLbBQOTEykeIrnIvIiIiojphIHJiHCEiIiKyDQYiJ9b4j9t3XC8wwFhqsXM1REREzouByIkFeCuglJf9CnnZjIiIqPYYiJyYIAi8pxkREZENMBA5Oa0f5xERERHVFQORk7u1F1GxnSshIiJyXgxETk6r4W7VREREdcVA5OS4WzUREVHdMRA5Oe5FREREVHcMRE6OI0RERER1x0Dk5MpHiLLzS1Bq5uaMREREtcFA5OSC1SrIPQRYROBagcHe5RARETklBiIn5+EhIJR7EREREdUJA5EL4DwiIiKiumEgcgFcaUZERFQ3DEQugLtVExER1Q0DkQvgbtVERER1w0DkAjiHiIiIqG4YiFwA5xARERHVDQORCygfIbqaVwKLRbRzNURERM6HgcgFNPJRwUMASi0irhdyc0YiIqK7xUDkAuQyD4T4ch4RERFRbTEQuYjyeURXcrn0noiI6G4xELmIsEBvAED6TQYiIiKiu8VA5CKaBZbtRZR2s8jOlRARETkfBiIXERbwxwhRDgMRERHR3WIgchHN/rhkxhEiIiKiu8dA5CLK5xBl5BRzLyIiIqK7xEDkIhprPCHzEGAstSA7n3sRERER3Q0GIhchl3lA51+29J7ziIiIiO4OA5ELkeYR3WAgIiIiuhsMRC6EK82IiIhqh4HIhYRxpRkREVGt1CoQffbZZ/j++++l5/PmzYO/vz969eqF1NRUmxVHd0daacbdqomIiO5KrQLRkiVL4OVVtjPy3r17sXLlSixduhTBwcGYOXOmTQukmuNeRERERLUjr82b0tPT0bp1awDA5s2b8cgjj+Dpp59G79690a9fP1vWR3chLKAspF7NL0GJyQxPhczOFRERETmHWo0Q+fj44MaNGwCAbdu2ITY2FgDg6emJ4mJerrGXQLUS3koZRBG4zLveExER1VitRogGDhyIJ598El27dsXZs2fx4IMPAgBOnDiBFi1a2LI+uguCIKBZoDdOZ+Uj/WYRWjXysXdJRERETqFWI0Tvv/8+evbsiWvXruHrr79GUFAQACApKQljx461aYF0d5qWL73nPCIiIqIaq9UIkb+/P1auXFnh+KuvvlrngqhuyidWp+fwkhkREVFN1WqEaOvWrUhMTJSev//+++jSpQvGjRuHnJwcmxVHdy8ssGxiNUeIiIiIaq5WgWju3LnIy8sDABw7dgyzZ8/G0KFDcfHiRcyaNcumBdLd4dJ7IiKiu1erS2YpKSno0KEDAODrr7/GsGHDsGTJEhw+fBhDhw61aYF0d8o3Z+QIERERUc3VaoRIqVSiqKjsC3fHjh0YNGgQACAwMFAaObKVy5cv44knnkBQUBC8vb3RpUsXJCUlSa+LoohFixZBp9PBy8sL/fr1w4kTJ6zOYTAYMH36dAQHB0OtVmPEiBHIyMiwaZ2Oovx+ZnklpdAXmexcDRERkXOoVSDq06cPZs2ahddeew0HDhyQlt2fPXsWTZs2tVlxOTk56N27NxQKBX788UecPHkSb7/9Nvz9/aU2S5cuxbJly7By5UocPHgQWq0WAwcORH5+vtQmLi4OmzZtwvr165GYmIiCggIMGzYMZrPZZrU6Ci+lDME+KgC8bEZERFRTtQpEK1euhFwux1dffYVVq1ahSZMmAIAff/wRQ4YMsVlxb775JsLCwrB69Wrce++9aNGiBQYMGIBWrVoBKBsdeuedd7Bw4UKMHj0akZGR+Oyzz1BUVIQvvvgCAKDX6/Hpp5/i7bffRmxsLLp27Yp169bh2LFj2LFjh81qdSTlE6szeNd7IiKiGqlVIGrWrBm+++47HD16FFOmTJGOL1++HO+9957NituyZQtiYmLw6KOPIiQkBF27dsXHH38svZ6SkoKsrCzpkh0AqFQq9O3bF3v27AFQtjeSyWSyaqPT6RAZGSm1qYzBYEBeXp7Vw1k08S8LRNytmoiIqGZqNakaAMxmMzZv3oxTp05BEAS0b98eI0eOhExmu/tnXbx4EatWrcKsWbPwt7/9DQcOHMCMGTOgUqkwceJEZGVlAQBCQ0Ot3hcaGorU1FQAQFZWFpRKJQICAiq0KX9/ZeLj4512X6XyzRkzuBcRERFRjdQqEJ0/fx5Dhw7F5cuX0bZtW4iiiLNnzyIsLAzff/+9dEmrriwWC2JiYrBkyRIAQNeuXXHixAmsWrUKEydOlNoJgmD1PlEUKxy73Z3aLFiwwGoLgby8PISFhdWmGw2uSUD5JTMGIiIiopqo1SWzGTNmoFWrVkhPT8fhw4dx5MgRpKWlITw8HDNmzLBZcY0bN5aW95dr37490tLSAABarRYAKoz0ZGdnS6NGWq0WRqOxwoaRf25TGZVKBT8/P6uHs2jKS2ZERER3pVaBKCEhAUuXLkVgYKB0LCgoCG+88QYSEhJsVlzv3r1x5swZq2Nnz55F8+bNAQDh4eHQarXYvn279LrRaERCQgJ69eoFAIiOjoZCobBqk5mZiePHj0ttXE3TAE6qJiIiuhu1umSmUqmslrWXKygogFKprHNR5WbOnIlevXphyZIlGDNmDA4cOICPPvoIH330EYCyS2VxcXFYsmQJIiIiEBERgSVLlsDb2xvjxo0DAGg0GkyZMgWzZ89GUFAQAgMDMWfOHERFRSE2NtZmtTqS8ktm+SWlyCsxwc9TYeeKiIiIHFutAtGwYcPw9NNP49NPP8W9994LANi/fz+effZZjBgxwmbFdevWDZs2bcKCBQuwePFihIeH45133sH48eOlNvPmzUNxcTGef/555OTkoHv37ti2bRt8fX2lNsuXL4dcLseYMWNQXFyMAQMGYM2aNTadAO5IvJVyBHgrkFNkwuWcYvg1ZiAiIiKqjiCKoni3b8rNzcWkSZPw7bffQqEo+7I1mUwYOXIkVq9ebbVxoqvIy8uDRqOBXq93ivlEw1ck4thlPT6eGIOBHaqeK0VEROTKavr9XasRIn9/f3zzzTc4f/48Tp06BVEU0aFDB7Ru3brWBZNtNfH3wrHLelzmPCIiIqI7qnEgutNd7Hft2iX9e9myZbUuiGyjfB4RV5oRERHdWY0D0ZEjR2rU7k77/1DDaMq9iIiIiGqsxoFo586d9VkH2Rhv30FERFRztdqHiBwfd6smIiKqOQYiF1V+P7ObhUYUGUvtXA0REZFjYyByURovBXxVZVdEr/CyGRERUbUYiFxY+WWzdF42IyIiqhYDkQsrX2l2mYGIiIioWgxELowrzYiIiGqGgciFcaUZERFRzTAQubDylWa8fQcREVH1GIhcWPklM44QERERVY+ByIWVXzLLzjfAUGq2czVERESOi4HIhQWplfBUlP2KM3NL7FwNERGR42IgcmGCIEjziNJuch4RERFRVRiIXFyLIDUA4NKNQjtXQkRE5LgYiFxcy0ZlgejiNQYiIiKiqjAQubiWwX8EousMRERERFVhIHJx4eWB6FqBnSshIiJyXAxELq5lIx8AZbfvKDFx6T0REVFlGIhcXLCPEr6ecogikHqDK82IiIgqw0Dk4gRBkOYRpVznZTMiIqLKMBC5gfLLZhe40oyIiKhSDERuIFwaIWIgIiIiqgwDkRu4tRcRL5kRERFVhoHIDXCEiIiIqHoMRG6gPBDlFJmQU2i0czVERESOh4HIDXgr5Wis8QTAHauJiIgqw0DkJjiPiIiIqGoMRG6iZXDZ0nvOIyIiIqqIgchN3LqnGQMRERHR7RiI3ET5JTOOEBEREVXEQOQmpEtmNwphtoh2roaIiMixMBC5iSYBXlDKPGAsteBKbrG9yyEiInIoDERuQuYhoK3WFwCwbl+qnashIiJyLAxEbmTmwAgAwCeJKThxRW/naoiIiBwHA5Eb+Uu7UDwY1Rhmi4i/bTzGuURERER/YCByM38f3gG+KjmOZuixdu8le5dDRETkEBiI3EyonyfmPdAOAPDWT2dwo8Bg54qIiIjsj4HIDY2/txnahPqg0GjGr+eu2bscIiIiu2MgckMeHgL6tmkEADiQctPO1RAREdkfA5Gbujc8CACwn4GIiIiIgchddWsRAKDs3mbXOY+IiIjcHAORm/L3VqLdHxs1HuQoERERuTkGIjd2b3ggAF42IyIiYiByY+WBiBOriYjI3TEQubF7W5QFolNZedAXm+xcDRERkf04VSCKj4+HIAiIi4uTjomiiEWLFkGn08HLywv9+vXDiRMnrN5nMBgwffp0BAcHQ61WY8SIEcjIyGjg6h1PiJ8nWgR5QxSBpFSOEhERkftymkB08OBBfPTRR+jUqZPV8aVLl2LZsmVYuXIlDh48CK1Wi4EDByI/P19qExcXh02bNmH9+vVITExEQUEBhg0bBrPZ3NDdcDi3Lpvl2LkSIiIi+3GKQFRQUIDx48fj448/RkBAgHRcFEW88847WLhwIUaPHo3IyEh89tlnKCoqwhdffAEA0Ov1+PTTT/H2228jNjYWXbt2xbp163Ds2DHs2LHDXl1yGOX7ER1IuWHnSoiIiOzHKQLR1KlT8eCDDyI2NtbqeEpKCrKysjBo0CDpmEqlQt++fbFnzx4AQFJSEkwmk1UbnU6HyMhIqU1lDAYD8vLyrB6uqPsfI0S/Z+hRbOSIGRERuSeHD0Tr16/H4cOHER8fX+G1rKwsAEBoaKjV8dDQUOm1rKwsKJVKq5Gl29tUJj4+HhqNRnqEhYXVtSsOqWmAFxprPFFqEbGPo0REROSmHDoQpaen44UXXsC6devg6elZZTtBEKyei6JY4djt7tRmwYIF0Ov10iM9Pf3uincSgiCgf7sQAMCOk1ftXA0REZF9OHQgSkpKQnZ2NqKjoyGXyyGXy5GQkID33nsPcrlcGhm6faQnOztbek2r1cJoNCInJ6fKNpVRqVTw8/Ozeriqge3Lfg47Tl2FxSLauRoiIqKG59CBaMCAATh27BiSk5OlR0xMDMaPH4/k5GS0bNkSWq0W27dvl95jNBqRkJCAXr16AQCio6OhUCis2mRmZuL48eNSG3fXs1UQvJUyXM0z4Nhlvb3LISIianByexdQHV9fX0RGRlodU6vVCAoKko7HxcVhyZIliIiIQEREBJYsWQJvb2+MGzcOAKDRaDBlyhTMnj0bQUFBCAwMxJw5cxAVFVVhkra78lTI0LdNI/x4PAs7Tl1F5zB/e5dERETUoBw6ENXEvHnzUFxcjOeffx45OTno3r07tm3bBl9fX6nN8uXLIZfLMWbMGBQXF2PAgAFYs2YNZDKZHSt3LAM7hOLH41nYfvIqZg9qa+9yiIiIGpQgiiInjdRAXl4eNBoN9Hq9S84nyi0yIvofO2C2iNg9rz/CAr3tXRIREVGd1fT726HnEFHD8fdWoluLsq0JtnG1GRERuRkGIpLE/rHabNuJLGTnlSApNQcXrxXYuSoiIqL6x0tmNeTql8wAIO1GEe5/a6fVMaXMA7vm9oPO38tOVREREdUeL5nRXWsW5C3dysNDAOQeAoxmCw6n8cavRETk2hiIyMraKd2ROL8/Tr/2AB6NKbtdyalM17yPGxERUTmnX3ZPtqWUe6BpQNkKs/aNy7YuOJ2Zb8+SiIiI6h1HiKhK7RuXXWvlCBEREbk6BiKqUltt2QjRFX0J9EUmO1dDRERUfxiIqEp+ngo0DShbXXYqi6NERETkuhiIqFrttGWXzU7zshkREbkwBiKqVoc/Jlaf4sRqIiJyYQxEVK12f0ysPs1LZkRE5MIYiKha5SvNzlzNh9nCTc2JiMg1MRBRtZoFesNLIUOJyYJLNwrtXQ4REVG9YCCiask8BGn5PfcjIiIiV8VARHfEHauJiMjVMRDRHXHHaiIicnUMRHRH0l5EWRwhIiIi18RARHfU7o9LZpdzi7HrTDZEkavNiIjItTAQ0R35eSoQ0zwAADB59UGM+3g/fs/ItW9RRERENsRARDXy0cQY/LV3OJQyD+y9eAOPrNqLs1d5CY2IiFwDAxHVSKBaib8P74Bf5vTFveGBMJotePPH0/Yui4iIyCYYiOiuNA3wRvzoKMg8BPx8Ohv7L96wd0lERER1xkBEd61VIx883i0MABD/42lOsiYiIqfHQES18kJsBLyVMiSn5+LH41n2LoeIiKhOGIioVkJ8PfHUfS0BAEu3nkax0WznioiIiGqPgYhq7an7W6KRrwqXbhRh2heHUWq22LskIiKiWmEgolrzUcnx4RP3QCX3wM+ns7Fw03HOJyIiIqfEQER1Et08ECvH3QMPAdhwKB1vbD0NQykvnxERkXNhIKI6G9ghFP8YFQUA+FfCRfR+4xe8ve0MsvNK7FwZERFRzTAQkU2M694M8aOj0FjjiesFRqz45TxilyUg9UahvUsjIiK6IwYispmx9zbD7nn98cH4e9Am1Ad5JaWYuSGZk62JiMjhMRCRTcllHhga1Rj/ntwNvio5Dqfl4oNdF+xdFhERUbUYiKheNA3wxmujIgEA7/58DsnpufYtiIiIqBoMRFRvRnbRYXhnHcwWEXHrjyC/xGTvkoiIiCrFQET1RhAE/GNkJHQaT1y6UYSZG47CYuE+RURE5HgYiKheabwVWPVENJRyD+w4dRUrfjlv75KIiIgqYCCietc5zB+v/zGfaPmOs9h0JAPpN4two8AAM0eMiIjIAcjtXQC5h0djwnD8sh6f7U3FzA1HpePhwWp8M603/DwVdqyOiIjcHUeIqMG8NKwDHr6nKQLVSngqyv6nl3K9EEu3nrZzZURE5O44QkQNRiHzwNtjOkvP91y4jnEf78fn+9PwUNemiG4eYMfqiIjInXGEiOymV6tgPBLdFKII/G3jMZi4ozUREdkJAxHZ1cKh7RGoVuLM1Xx8sPMCRJGTrImIqOExEJFdBaiVWDi0PYCyFWi93/gFi7acwNmr+XaujIiI3AkDEdnd6Hua4Kn7wuGtlOGKvgRr9lzC6A/24HqBwd6lERGRm2AgIrsTBAELH+yAwy8PxCcTY9CqkRoFhlJsOJhu1U4URRQYSpGpL8a5q/nQF/FWIEREZBtcZUYOw1MhQ2yHUOQWmzDny6P4Yn8anu3bCjIPAYWGUjy8ag9OZ926lKaUe+ChLk0w5b5wtAn1tWPlRETk7DhCRA5nWKfGCPBW4HJuMX4+dRUAsHz7WSkMyT0E+HrKYSy1YMOhdAxa/ivmfHm0ulMSERFVy6EDUXx8PLp16wZfX1+EhIRg1KhROHPmjFUbURSxaNEi6HQ6eHl5oV+/fjhx4oRVG4PBgOnTpyM4OBhqtRojRoxARkZGQ3aF7oKnQobHujUDAPxnbypOXNFj9Z5LAIDVk7vh3OsP4PdXBuGrZ3vigUgtAOCrpAzOOSIiolpz6ECUkJCAqVOnYt++fdi+fTtKS0sxaNAgFBYWSm2WLl2KZcuWYeXKlTh48CC0Wi0GDhyI/Pxbl1bi4uKwadMmrF+/HomJiSgoKMCwYcNgNpvt0S2qgfHdm0EQgMTz1zH9v0dgtoh4sFNj9G8XAkEQIAgCYloEYtUT0Wgd4gMASE7LtW/RRETktATRiTZ+uXbtGkJCQpCQkID7778foihCp9MhLi4O8+fPB1A2GhQaGoo333wTzzzzDPR6PRo1aoS1a9fiscceAwBcuXIFYWFh+OGHHzB48OBKP8tgMMBguDXikJeXh7CwMOj1evj5+dV/ZwlPfnYQO05lAwB8VXL8PLsvQvw8K7Sb8+VRfJWUgWn9W2PO4LYNXSYRETmwvLw8aDSaO35/O/QI0e30ej0AIDAwEACQkpKCrKwsDBo0SGqjUqnQt29f7NmzBwCQlJQEk8lk1Uan0yEyMlJqU5n4+HhoNBrpERYWVh9dompM6NlC+vfcIW0rDUMA0CXMHwCQnJ5b/0UREZFLcppAJIoiZs2ahT59+iAyMhIAkJWVBQAIDQ21ahsaGiq9lpWVBaVSiYCAgCrbVGbBggXQ6/XSIz09vcq2VD/ua112a49HoptifPfmVbYrD0RH03NhsTjNgCcRETkQp1l2P23aNPz+++9ITEys8JogCFbPRVGscOx2d2qjUqmgUqlqVyzZhIeHgH8+2vmO7dppfeGp8EC+oRQXrxegdQiX4BMR0d1xihGi6dOnY8uWLdi5cyeaNm0qHddqy1YY3T7Sk52dLY0aabVaGI1G5OTkVNmGnJtc5oGoJhoAwBFOrCYiolpw6EAkiiKmTZuGjRs34pdffkF4eLjV6+Hh4dBqtdi+fbt0zGg0IiEhAb169QIAREdHQ6FQWLXJzMzE8ePHpTbk/Lo2K7skynlERERUGw59yWzq1Kn44osv8M0338DX11caCdJoNPDy8oIgCIiLi8OSJUsQERGBiIgILFmyBN7e3hg3bpzUdsqUKZg9ezaCgoIQGBiIOXPmICoqCrGxsfbsHtkQJ1YTEVFdOHQgWrVqFQCgX79+VsdXr16NyZMnAwDmzZuH4uJiPP/888jJyUH37t2xbds2+PremkeyfPlyyOVyjBkzBsXFxRgwYADWrFkDmUzWUF2helYeiE5n5aPYaIbMQ8AbP56Gp8IDz/VrBV9PhX0LJCIih+ZU+xDZU033MSD7EEUR3Zf8jOx8A758tie+O3oFn+1NBQCE+qnw6oiOGNxRe8fJ9kRE5Fpcch8ioqoIgiCNEr3542kpDDXx98LVPAOeXXcYz6xNQk6h0Y5VEhGRo2IgIpfRpZk/AOBQatmKwhkDIvDz7L6Y8ZfWUMgEbDt5FUPf240DKTftWCURETkiBiJyGeUjRADQv20jxA2IgKdChlmD2mLz1N5oGaxGpr4Ej3+0F698cxz7L95Aqdliv4KJiMhhcA5RDXEOkeMrNpoRuywBvp5ybHi6JzTe1hOpCw2lePmb49h4+LJ0TOOlwNP3t8TU/q0bulwiImoANf3+ZiCqIQYi51BqtkAEoJBVPfi583Q2vj16BTvPZCOnyAQA+GRiDGI7cKNOIiJXw0nV5JbkMo9qwxAA9G8XgmWPdcGhlwbi/3q3AAAs2HQMuUXVT7jOKTTi418vIvHcdVuVS0REDsKh9yEiqk8yDwHzh7TDr2ev4cK1QizacgLvPN4VqTcKsfHwZVhEEe20fmge5I3vj2XiP3suodBoBgDMjG2D6X9pDQ8P62X8Z6/m49+JKejdOhjDO+vs0S0iIqoFXjKrIV4yc11H0nLw8Ko9sIhAtxYBOJSag6r+q2ji74XLucUAgAcitXhpWAd4K2QwlFqwcuc5/PdAOswWETIPARue7oGYFoEN2BMiIrod5xDZGAORa1u69TQ+2HVBen5/m0Zo7OeJ01fzcSG7AK1CfDCtf2vEtg/Bl4cysHDzMZjMlf+nUx6aGms88cOM+xCgVjZUN4iI6DY1/f7mJTMiAC/ERqDQUAql3APjujdHeLC6yrZjuoWhVYgac7/8Hak3i2C2lAWjqCYa/G1oe0Q11WDEikRcvF6I2V8exScTYypcWiMiIsfCEaIa4ggRVaXUbIHJLMJLeeveeCev5GHUB7/BWGrBuO7N8FzfVggL9LZjlURE7omrzIgaiFzmYRWGAKCDzg+vDO8AAPhifxruW7oTT3yyH9tOZMFiufX/QW4WGrFuXyouXito0JqJiMgaL5kR1ZPx3ZujkY8Ka/elYve560g8X/ZoG+qLJ+8Lx/HLemw4lI4SkwWNNZ7YNvN++Hoq7nxiIiKyOV4yqyFeMqO6SL9ZhC8OpGHt3lQUGEqtXlPIBJjMIsZ1b4YlD0VVeO/Zq/lY8sMpDO+kw8PRTSs9f6nZgu+PZaJnqyCE+HrWSx+IiJwRV5nZGAMR2YK+yITP9l7CpiOXERbojWfvbwlBEDD2430AgP8+1QM9WwVJ7c9k5WPcx/two9AIDwH4dHI39G8bUuG85avk2ob64ptpveGpkFVoQ0TkjhiIbIyBiOrT3zYdwxf709A8yBtbX7gfXkoZzmTlY+zH+3Cz0Ai1UoZCoxm+Kjk2Te2N1iE+0nvPZOXjwfd2o/SPuUl/7R2Ov/8xf4mIyN1xUjWRE1nwQDs01ngi9UYRer7xM3rF/4xR7/+Gm4VGRDXRYNfc/ujWIgD5hlI8/Z9D0P9xDzaLRcTCTcdQahHRNtQXAPDv31Lw69lr9uwOEZHTYSAicgC+ngrEj46ChwDkFplwRV+CYpMZnZpqsG5KdzTyVWHVE9HQaTxx8Xohhr63G/87lI7PD6ThUGoOvJUyrP6/bpjQozkAYM6XR3GzsPp7sxER0S28ZFZDvGRGDSFLX4KbhUaUWiwAgA6N/SD/081qT17Jw/+tOYCreQar9730YHs8eV9LFBvNGLZiNy5cK0SAtwJjuoVh/L3NofFW4GahETcLDbheYMTNQiPyik0I9fNEi2A1wgK8UGQ041qBAfklpeio80Owj6pB+05EVB84h8jGGIjIURQbzVi3LxWrEi7gZqERHXV++GZqbyk4nc7Kw5OfHUJGTnGtP0MQgM5N/TGgXQjGdW+GoD+Fo0OXbmLVrgsoMprh4QF4KeR4okcz9KtksjcRkb0xENkYAxE5mgJDKX45nY3erYKsAgtQtgz/l9PZ0h5IAKBWyhDoo0SQWoUgtRK+nnJk6kuQcr0Q2fkGqOQeCPZRQSX3wMXrhdK5QnxVeOexLujVOhhfJWVgwcbfK72P2+h7muDvwzpA46XA1TwDrhcY0CbUF0r5na/MH7+sx7YTWZjcOxyBvPcbEdkQA5GNMRCRs8orMUEp86h2Kb6x1AKFTIAglN1zLUtfgp1nsvFpYgrOZxdAEID7IhpJk7WHdNRiaKfGEEURR9Jy8dneSxBFwNdTDotFRKHRDADQeCnwQKQWI7ro0KtVcIXPFUURn+9Pw+JvT8JotuAv7ULw6aQYqY61ey/hk8QULBzaHoM6am3y8xBFEXklpdB4cRNMInfAQGRjDETkjoqMpVj87UmsP5guHZv+l9aYGdvG6oa1h9NyMO+r33E+u+wWJDIPAd4KGfL/tAnlvCFt8Xy/1tLzYqMZCzcfw8bDl60+c+W4rhjWSYcDKTfx+Ed7YRHLNq/8aGJMpXsw3Q2T2YKn/3MICWevYcaACMz4SwRvvEvk4hiIbIyBiNzZt0ev4D97L+GJHs0xskuTStsYSs04mq5HoFqBZoFqyDwE7E+5ga+SMrDx8GUoZAJ+mHEfIkJ9YbaIeOo/h/DL6Wx4CMD8Ie1QZDTj3Z/PIdhHha+f64mxH+3DFX0JgtRK3Cg0QiX3wOrJ3dCrdcWRppoQRRF/23QM/z1wK9wNaBeCZY914WgRkQtjILIxBiKi2hFFEU9+dgg/n85G12b++OrZXli2/Qze33nBKuQYSs148L1EnM8ugI9KjgJDKcKD1dj0fC/M+fJ37Dh1FV4KGT776724NzxQOn9Sag42HcmAn6cCjTWeaKzxglbjicYaTwSqldLlt49/vYjXfzgFQQAm92qBz/enwVhqQXiwGuue7I4m/l72+hERUT1iILIxBiKi2svUF2Pgsl9RYCjFoA6h2HbyKgDgnce6YFTXWyNOBy/dxKMf7gUAKGUe2Ph8L0Q20aDEZMZT/zmE3eeuw0shw78nd0PPVkH47vcrmLkhudJJ3uXn0Go8EeKrQlJaDkQReHlYB0zpE45jGXo8uy4Jl3OLERHig6+e7QWNd/UjRRaLiJQbhTCYLDCZy27KG+J39/eOu5ZvwI1CA9pp+beEqL4xENkYAxFR3Xy+PxULNx2Xnj91XzgWPljxFiOLtpzAmj2XsHhkR0zs2UI6/udQ5KnwwGMxYfjPvlSIItCvbSM0D/RGpr4EWXklyNSX4HqBAbf/dZvQozkWj+wojRpl6ovx0Pt7kJVXgnvDA7F2yr1QySuffH69wIApaw7iaIZeOqaQCfjwiWgMaB8KoGx13wsbknEw5Sae6dsKE3o0r7DK7kxWPh77aC9yi0x4bVSktJnmn5WaLdh+8irOXM2HSi6DSu6Blo3U6NumkVQ7EdUMA5GNMRAR1Y3FImLsx/uwP+Um7osIxurJ3aw2nSwniiKu5RsqHXkpMZnx3Lok7Dxz69YkE3s2xyvDO0J22+RoY6kF2fklyNKXBSSLKGJYJ12Fdqez8vDoqr3IN5Sif9tGiGrqj7xiE5RyD4zorENkEw0ycoow8dMDuHi9EEqZBzTeCoiiiOsFRvh6yvHttD5oEazGq9+ewOrfLknnbh7kjbmD22JIRy3kMg+k3ijEIx/uxbX8WxtrLnkoCuO6NwNQtpXCV4fS8UliSqX7SEU3D8DCB9vjnmYB1f6si41miBDhrZRbHTeZLbCIYoXQd+KKHh6CgPaN+beNXA8DkY0xEBHVXU6hEd8fy8Sork3go5Lf+Q2VMJSa8cJ/k7HtZBZmDWyDqf1b13nUZM/565i0+kCll966hPnj6h+jTk38vbB2yr1o2cgHxlILxn68D0mpOWgb6ovHuoVh8XcnAQD/17sFvj2aiesFZcEn1E+FR6Kb4pvkK8jIKUY7rS/uDQ/Ef/amAgCe6NEMF68V4uClm1INgWolYtuHwCKWBZxfTmej2FS2ncEj0U3x5sOdKoQ7oOxnPOL9RFzLN+CJ7s3xTN9WUMgE/Pu3S1j9Wwr8PBXYNLUXQnzLAmdSag7G/GsvRFHE8se6VDlp/m5YLCKOpOci1E+FpgHedT4fUV0wENkYAxGRY8kvMcHX03arw347fx1fH86Al0IGjZcCaTeL8NOJLCmgtA7xwdop96Kx5tbk66t5JRi2ItFqxGfWwDaYMSACBYZSfPzrRazdl2p1X7kWQd7437M90chHhde+O4V//5ZiVUfLYDX+r084HrmnKbyUMqvPenvbGXyZlAFRBF4b2RET/nRJESgbXXv+88P48XiWdMxT4QG5hwcK/rQFQo+Wgfj8yR4oNpkx9N3dSLtZBADwEIC3HumMh6ObwmwRcSozD0E+Sqs+l5ot+CopA4FqZYW9oc5nF+CrpAxsSb6MK/oSeCo88Mrwjni8Wxgv9ZHdMBDZGAMRkfu5lm/A/w6l40puMeYMaouASnbRPpByE+M+3odSi4jhnXV47/EuVl/+hlIztp24ig0H01FgKMXKcV2lURNRFLHyl/M4dlmPnq2CcH+bRmgZrK42PHy25xJe2XICfp5y/DKnn9U9575KysCcL49C7iHg78M74OukDGnOUzutL8b3aI43fjiFQqMZ0//SGpn6EnyVlIEm/l7o1SoIXyZlQBCAvm0a4UhaLvTFJsg9BEzpE44ZAyJwNa8Es/53FMnpuQDKRsIWDm0PQRCwatd5LN9xDmZL2VeKQiZIYfKBSC3mDG4LX085vBQymwbZqhQbzdh4JAMRIb7o1iKAgcyNMRDZGAMREVVl55lsJKfl4rl+rardEdwWzBYRI1Ym4sSVPDx8T1O8PaYzACD9ZhEeeHc3CgylmDu4Lab2bw1RFHEg5SbMooge4UHw8BDwTfJlvLA+WTqfIADrn+qBbi0C8cqWE1i7L1V6zVspQ9Efu46H+qmgLzahxGSxOn5fRDCMpRbsT7kJoCxMPd4tDP3bheA/ey/hrZ/OVLgUeU8zfyx/rAuaB6kBlAXPz/ZcgiAAvVsH455mATW65UtVklJzMPfLo9ItaKKbB2Bq/1bo3zakRsGo2GjGiSt6RIT43nHlITk+BiIbYyAiIkdxOC0Hoz/YAwD49+QYZOcZ8GliCs5lFyC6eQD+90zPSucXlXtp8zGs25cGAHi+XyvMG9IOQNmI1dp9qdAXmdA7Ihidmmjw67lreGXLCaTfLJvk3atVEP75aGccTc/FrP8dleY1qZUyvDYqEqPvaWr1Wb9n5GLhpuO4eK0ARSaztPLPVyXH66OjoC82YenW08gvuXVJz1spQ1iAN9QqGdQqOUxmCwoNZhQaSyH3EKBWyaFWyhHip0J4kBrNgspG3PJKSnEmKw9f7E+DRQSC1ErkG0phLLUAADqH+eOV4R0qnZSeV2LCvgs38P2xTGw/eRVFRjMCvBV48YF2eDQ6rMKO5qIo4sSVPJzJykfqjbL7AfZqHYyhkdpKFwv82YaDafhkdwqm9AnH4/c2k44nnruODxMu4IkezTEk0ja3qiEGIptjICIiR/Li179b3VIFAPw85fhu+n1SQKhKicmMF9YfgYcg4N3Hu95xNKbEZMa6fanw81TgkeimUjg4eSUPcRuOwN9biaUPd0KLYHW15xFFERk5xZi5IRmHUnOsXots4oeWwT747fx13PjTnKvaeviepvj7sA4wlJrxaWIK1u1Lle6xN6qLDl3C/JFTZMK1AgOS03JxKivPapsGL4VMCnvRzQPw6oiOiGyiAVB2S5sFG4/hm+QrFT63WaA3pvQJRyNfFXKKjCgymBHVVIPo5gEwW0Qs2nLC6vf2j1GReKJHc/x86iqeW3cYRnNZeJs/pB2e7duywohWdl4Jtp7IwuCOWoTWYg+syvxy+iq+TroMk9kCD0FAsK8S0/pHQKuxzfntjYHIxhiIiMiR5BQaMXB5Aq4XGNEm1AcPdW2K0fc0sdmXZH0qNVuwfMdZfLDrAnyUcswZ3BZP9GgOmYcAi0XE+WsFuJZvQIGhFIWGUshlHvBRyeCtvHXz4AKDCVdyS3DpeiFSbxZBJgjQeCmg8VJgSJS2wn3vsvNL8NbWsknpVWkR5I3+7UIworMOHXUafLbnEpbvOCtdHnywU2M8Gt0US344hbNXCyDzENA9PBAtgtXwVsjw9eEM5BSZKj23n6ccgWolLt0ogiAAPVsGYc+FGwCAsfc2w1dJ6TCZRbQMVkuX+sbENMWrIyKlyfUHUm7i+c8P43qBAb4qORYMbY/Hu4WhwFiKrcezsPvcdVzVlyA7vwQlJgtGdW2Cp+9vicBK5r4BZds8/OO7kxWCNQC0bKTG/57paTVHzZbSbxbh/Z3nkXj+OkpMFhhLzTCUWrD6/7pVeiPoumAgsjEGIiJyNNn5JcgrNqFVIx+nnDR86Xoh/L0V8Peu/Au7PvyekYtPdqeg1GKBv7cSgd5KtG/sh27hAdJWBH92JbcYb249jS1Hr1iNIIX4qrBy3D1Wt5EpMpZiw8F0fJN8BXIPAf7eSihkAvZdvCEFJX9vBd57vCvuiwjG69+fwieJt1YZDu+sw7IxnfHF/jS8+u0JWETA11OOEZ110Pp54t2fz6HUIlqNXrUO8UH6zSIY/rgseDu1UobxPZojSK2EodQCQ6kZBpMFhlILEs5eQ9rNsoA2sUdztNH6wmIRsWrXBVzRl6B9Yz+sf6oHRIjYdOQyDqflwtdTjiC1Ep4KGa4XGHAt3wBBEDCxZ3N0axFY4fOLjKXYcSobZ7LyoFbJ4eepwPHLenyVlIFSS8X4sfr/utX5Js63YyCyMQYiIiL3dSozD29vO4Mdp7LRo2Ug3hvbtdIAVRmzRURyeg6OpOXigajG0n3zRFHEkh9O4ePdKXg0uine+NPeUjvPZOPlzccrbNA5vLMOSx6KxJeHMvDPbWek0avWIT4Y3kmHViFqhPh64mahAe/9fB4nM/Oqra2JvxfeHtMZPVoGSccuXivAmH/tw/UCA5r4e+FagUGah1Wd2PYheOq+ligwlCL1RhGSUnPw8+mrKDFV/t77IoLx5H0t0chHBZXCA0qZBxr5qmy+MIGByMYYiIiI6HqBAUF/ummwLeiLTJWuZrNYROy7eAP/O5SO/Sk38eR9LfHX3i2kz87IKcLuc9cR1USDjjq/CjVZLCK2nczCthNXIQiCFDpUCg+o5DIEeiswOrop/CrZBuF0Vh4e+9c+6IvLRrY6NPbD0CgtTGaxbG6U0YwgHyVCfD1xPjsf/zuUIW25cLtmgd7oExEMU6kFeSUmqOQyTOzZHDGVjCjVBwYiG2MgIiIid3ImKx8/HMvEwA6h0oTyqly4VoC3t53B/os3EernieZB3mjVyAeDOoYiqonGrpd0GYhsjIGIiIjI+dT0+7v2O18RERERuQgGIiIiInJ7DERERETk9hiIiIiIyO0xEBEREZHbYyAiIiIit8dARERERG7PrQLRBx98gPDwcHh6eiI6Ohq7d++2d0lERETkANwmEG3YsAFxcXFYuHAhjhw5gvvuuw8PPPAA0tLS7F0aERER2Znb7FTdvXt33HPPPVi1apV0rH379hg1ahTi4+Pv+H7uVE1EROR8uFP1nxiNRiQlJWHQoEFWxwcNGoQ9e/ZU+h6DwYC8vDyrBxEREbkmtwhE169fh9lsRmhoqNXx0NBQZGVlVfqe+Ph4aDQa6REWFtYQpRIREZEduEUgKnf73XZFUazyDrwLFiyAXq+XHunp6Q1RIhEREdmB3N4FNITg4GDIZLIKo0HZ2dkVRo3KqVQqqFSqhiiPiIiI7MwtApFSqUR0dDS2b9+Ohx56SDq+fft2jBw5skbnKJ97zrlEREREzqP8e/tOa8jcIhABwKxZszBhwgTExMSgZ8+e+Oijj5CWloZnn322Ru/Pz88HAM4lIiIickL5+fnQaDRVvu42geixxx7DjRs3sHjxYmRmZiIyMhI//PADmjdvXqP363Q6pKenw9fXt8p5R84qLy8PYWFhSE9Pd4stBdhf1+VOfQXYX1fnTv2tz76Kooj8/HzodLpq27nNPkRUNXfbY4n9dV3u1FeA/XV17tRfR+irW60yIyIiIqoMAxERERG5PQYigkqlwiuvvOI22wywv67LnfoKsL+uzp366wh95RwiIiIicnscISIiIiK3x0BEREREbo+BiIiIiNweAxERERG5PQYiNxEfH49u3brB19cXISEhGDVqFM6cOWPVRhRFLFq0CDqdDl5eXujXrx9OnDhhp4ptKz4+HoIgIC4uTjrmav29fPkynnjiCQQFBcHb2xtdunRBUlKS9Lqr9Le0tBQvvfQSwsPD4eXlhZYtW2Lx4sWwWCxSG2fu66+//orhw4dDp9NBEARs3rzZ6vWa9M1gMGD69OkIDg6GWq3GiBEjkJGR0YC9qLnq+msymTB//nxERUVBrVZDp9Nh4sSJuHLlitU5XKW/t3vmmWcgCALeeecdq+PO0t+a9PXUqVMYMWIENBoNfH190aNHD6SlpUmvN2RfGYjcREJCAqZOnYp9+/Zh+/btKC0txaBBg1BYWCi1Wbp0KZYtW4aVK1fi4MGD0Gq1GDhwoHQfN2d18OBBfPTRR+jUqZPVcVfqb05ODnr37g2FQoEff/wRJ0+exNtvvw1/f3+pjav0980338SHH36IlStX4tSpU1i6dCneeustrFixQmrjzH0tLCxE586dsXLlykpfr0nf4uLisGnTJqxfvx6JiYkoKCjAsGHDYDabG6obNVZdf4uKinD48GG8/PLLOHz4MDZu3IizZ89ixIgRVu1cpb9/tnnzZuzfv7/S2004S3/v1NcLFy6gT58+aNeuHXbt2oWjR4/i5Zdfhqenp9SmQfsqklvKzs4WAYgJCQmiKIqixWIRtVqt+MYbb0htSkpKRI1GI3744Yf2KrPO8vPzxYiICHH79u1i3759xRdeeEEURdfr7/z588U+ffpU+bor9ffBBx8U//rXv1odGz16tPjEE0+IouhafQUgbtq0SXpek77l5uaKCoVCXL9+vdTm8uXLooeHh7h169YGq702bu9vZQ4cOCACEFNTU0VRdM3+ZmRkiE2aNBGPHz8uNm/eXFy+fLn0mrP2t7K+PvbYY9J/t5Vp6L5yhMhN6fV6AEBgYCAAICUlBVlZWRg0aJDURqVSoW/fvtizZ49darSFqVOn4sEHH0RsbKzVcVfr75YtWxATE4NHH30UISEh6Nq1Kz7++GPpdVfqb58+ffDzzz/j7NmzAICjR48iMTERQ4cOBeBafb1dTfqWlJQEk8lk1Uan0yEyMtLp+w+U/e0SBEEa/XS1/losFkyYMAFz585Fx44dK7zuKv21WCz4/vvv0aZNGwwePBghISHo3r271WW1hu4rA5EbEkURs2bNQp8+fRAZGQkAyMrKAgCEhoZatQ0NDZVeczbr16/H4cOHER8fX+E1V+vvxYsXsWrVKkREROCnn37Cs88+ixkzZuA///kPANfq7/z58zF27Fi0a9cOCoUCXbt2RVxcHMaOHQvAtfp6u5r0LSsrC0qlEgEBAVW2cVYlJSV48cUXMW7cOOkGoK7W3zfffBNyuRwzZsyo9HVX6W92djYKCgrwxhtvYMiQIdi2bRseeughjB49GgkJCQAavq9ym5+RHN60adPw+++/IzExscJrgiBYPRdFscIxZ5Ceno4XXngB27Zts7oefTtX6a/FYkFMTAyWLFkCAOjatStOnDiBVatWYeLEiVI7V+jvhg0bsG7dOnzxxRfo2LEjkpOTERcXB51Oh0mTJkntXKGvValN35y9/yaTCY8//jgsFgs++OCDO7Z3xv4mJSXh3XffxeHDh++6dmfrb/kiiJEjR2LmzJkAgC5dumDPnj348MMP0bdv3yrfW1995QiRm5k+fTq2bNmCnTt3omnTptJxrVYLABVSd3Z2doX/N+oMkpKSkJ2djejoaMjlcsjlciQkJOC9996DXC6X+uQq/W3cuDE6dOhgdax9+/bSag1X+v3OnTsXL774Ih5//HFERUVhwoQJmDlzpjQS6Ep9vV1N+qbVamE0GpGTk1NlG2djMpkwZswYpKSkYPv27dLoEOBa/d29ezeys7PRrFkz6e9WamoqZs+ejRYtWgBwnf4GBwdDLpff8e9WQ/aVgchNiKKIadOmYePGjfjll18QHh5u9Xp4eDi0Wi22b98uHTMajUhISECvXr0autw6GzBgAI4dO4bk5GTpERMTg/HjxyM5ORktW7Z0qf727t27wjYKZ8+eRfPmzQG41u+3qKgIHh7Wf7pkMpn0/zhdqa+3q0nfoqOjoVAorNpkZmbi+PHjTtn/8jB07tw57NixA0FBQVavu1J/J0yYgN9//93q75ZOp8PcuXPx008/AXCd/iqVSnTr1q3av1sN3lebT9Mmh/Tcc8+JGo1G3LVrl5iZmSk9ioqKpDZvvPGGqNFoxI0bN4rHjh0Tx44dKzZu3FjMy8uzY+W28+dVZqLoWv09cOCAKJfLxddff108d+6c+Pnnn4ve3t7iunXrpDau0t9JkyaJTZo0Eb/77jsxJSVF3LhxoxgcHCzOmzdPauPMfc3PzxePHDkiHjlyRAQgLlu2TDxy5Ii0qqomfXv22WfFpk2bijt27BAPHz4s/uUvfxE7d+4slpaW2qtbVaquvyaTSRwxYoTYtGlTMTk52epvl8FgkM7hKv2tzO2rzETRefp7p75u3LhRVCgU4kcffSSeO3dOXLFihSiTycTdu3dL52jIvjIQuQkAlT5Wr14ttbFYLOIrr7wiarVaUaVSiffff7947Ngx+xVtY7cHIlfr77fffitGRkaKKpVKbNeunfjRRx9Zve4q/c3LyxNfeOEFsVmzZqKnp6fYsmVLceHChVZfkM7c1507d1b63+qkSZNEUaxZ34qLi8Vp06aJgYGBopeXlzhs2DAxLS3NDr25s+r6m5KSUuXfrp07d0rncJX+VqayQOQs/a1JXz/99FOxdevWoqenp9i5c2dx8+bNVudoyL4KoiiKth93IiIiInIenENEREREbo+BiIiIiNweAxERERG5PQYiIiIicnsMREREROT2GIiIiIjI7TEQERERkdtjICIiIiK3x0BERA6rRYsWeOedd2rcfteuXRAEAbm5ufVWExG5Ju5UTUQ2069fP3Tp0uWuQkx1rl27BrVaDW9v7xq1NxqNuHnzJkJDQyEIgk1quFu7du1C//79kZOTA39/f7vUQER3T27vAojIvYiiCLPZDLn8zn9+GjVqdFfnViqV0Gq1tS2NiNwYL5kRkU1MnjwZCQkJePfddyEIAgRBwKVLl6TLWD/99BNiYmKgUqmwe/duXLhwASNHjkRoaCh8fHzQrVs37Nixw+qct18yEwQBn3zyCR566CF4e3sjIiICW7ZskV6//ZLZmjVr4O/vj59++gnt27eHj48PhgwZgszMTOk9paWlmDFjBvz9/REUFIT58+dj0qRJGDVqVJV9TU1NxfDhwxEQEAC1Wo2OHTvihx9+wKVLl9C/f38AQEBAAARBwOTJkwGUBcGlS5eiZcuW8PLyQufOnfHVV19VqP37779H586d4enpie7du+PYsWN3/FwiqjsGIiKyiXfffRc9e/bEU089hczMTGRmZiIsLEx6fd68eYiPj8epU6fQqVMnFBQUYOjQodixYweOHDmCwYMHY/jw4UhLS6v2c1599VWMGTMGv//+O4YOHYrx48fj5s2bVbYvKirCP//5T6xduxa//vor0tLSMGfOHOn1N998E59//jlWr16N3377DXl5edi8eXO1NUydOhUGgwG//vorjh07hjfffBM+Pj4ICwvD119/DQA4c+YMMjMz8e677wIAXnrpJaxevRqrVq3CiRMnMHPmTDzxxBNISEiwOvfcuXPxz3/+EwcPHkRISAhGjBgBk8lU7ecSkQ2IREQ20rdvX/GFF16wOrZz504RgLh58+Y7vr9Dhw7iihUrpOfNmzcXly9fLj0HIL700kvS84KCAlEQBPHHH3+0+qycnBxRFEVx9erVIgDx/Pnz0nvef/99MTQ0VHoeGhoqvvXWW9Lz0tJSsVmzZuLIkSOrrDMqKkpctGhRpa/dXkN5nZ6enuKePXus2k6ZMkUcO3as1fvWr18vvX7jxg3Ry8tL3LBhwx0/l4jqhnOIiKhBxMTEWD0vLCzEq6++iu+++w5XrlxBaWkpiouL7zhC1KlTJ+nfarUavr6+yM7OrrK9t7c3WrVqJT1v3Lix1F6v1+Pq1au49957pddlMhmio6NhsViqPOeMGTPw3HPPYdu2bYiNjcXDDz9sVdftTp48iZKSEgwcONDquNFoRNeuXa2O9ezZU/p3YGAg2rZti1OnTtXqc4mo5njJjIgahFqttno+d+5cfP3113j99dexe/duJCcnIyoqCkajsdrzKBQKq+eCIFQbXiprL962uPb2FWm3v367J598EhcvXsSECRNw7NgxxMTEYMWKFVW2L6/v+++/R3JysvQ4efKk1TyiqpTXd7efS0Q1x0BERDajVCphNptr1Hb37t2YPHkyHnroIURFRUGr1eLSpUv1W+BtNBoNQkNDceDAAemY2WzGkSNH7vjesLAwPPvss9i4cSNmz56Njz/+GEDZz6D8POU6dOgAlUqFtLQ0tG7d2urx53lWALBv3z7p3zk5OTh79izatWt3x88lorrhJTMispkWLVpg//79uHTpEnx8fBAYGFhl29atW2Pjxo0YPnw4BEHAyy+/XO1IT32ZPn064uPj0bp1a7Rr1w4rVqxATk5OtfsYxcXF4YEHHkCbNm2Qk5ODX375Be3btwcANG/eHIIg4LvvvsPQoUPh5eUFX19fzJkzBzNnzoTFYkGfPn2Ql5eHPXv2wMfHB5MmTZLOvXjxYgQFBSE0NBQLFy5EcHCwtOKtus8lorrhCBER2cycOXMgk8nQoUMHNGrUqNr5QMuXL0dAQAB69eqF4cOHY/DgwbjnnnsasNoy8+fPx9ixYzFx4kT07NkTPj4+GDx4MDw9Pat8j9lsxtSpU9G+fXsMGTIEbdu2xQcffAAAaNKkCV599VW8+OKLCA0NxbRp0wAAr732Gv7+978jPj4e7du3x+DBg/Htt98iPDzc6txvvPEGXnjhBURHRyMzMxNbtmyxGnWq6nOJqG64UzUR0Z9YLBa0b98eY8aMwWuvvdZgn8sdronsi5fMiMitpaamYtu2bejbty8MBgNWrlyJlJQUjBs3zt6lEVED4iUzInJrHh4eWLNmDbp164bevXvj2LFj2LFjB+fmELkZXjIjIiIit8cRIiIiInJ7DERERETk9hiIiIiIyO0xEBEREZHbYyAiIiIit8dARERERG6PgYiIiIjcHgMRERERub3/B32IeKm+Cd77AAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import numpy as np\n",
    "from torch.utils.data import Dataset, DataLoader\n",
    "from torch.optim import SGD, Adam\n",
    "from copy import deepcopy\n",
    "\n",
    "# 将数据适配为PyTorch数据集\n",
    "class SeqLabelDataset(Dataset):\n",
    "    def __init__(self, tokens, labels):\n",
    "        self.tokens = deepcopy(tokens)\n",
    "        self.labels = deepcopy(labels)\n",
    "        \n",
    "    def __getitem__(self, idx):\n",
    "        return self.tokens[idx], self.labels[idx]\n",
    "    \n",
    "    def __len__(self):\n",
    "        return len(self.labels)\n",
    "    \n",
    "    # 将每个批次转化为PyTorch张量\n",
    "    @classmethod\n",
    "    def collate_batch(cls, inputs):\n",
    "        input_ids, labels, masks = [], [], []\n",
    "        max_len = -1\n",
    "        for ids, tags in inputs:\n",
    "            input_ids.append(ids)\n",
    "            labels.append(tags)\n",
    "            masks.append([1] * len(tags))\n",
    "            max_len = max(max_len, len(ids))\n",
    "        for ids, tags, msks in zip(input_ids, labels, masks):\n",
    "            pad_len = max_len - len(ids)\n",
    "            ids.extend([0] * pad_len)\n",
    "            tags.extend([0] * pad_len)\n",
    "            msks.extend([0] * pad_len)\n",
    "        input_ids = torch.tensor(np.array(input_ids),\\\n",
    "            dtype=torch.long)\n",
    "        labels = torch.tensor(np.array(labels), dtype=torch.long)\n",
    "        masks = torch.tensor(np.array(masks), dtype=torch.uint8)\n",
    "        return {'input_ids': input_ids, 'masks': masks,\\\n",
    "                'labels': labels}\n",
    "    \n",
    "# 准备数据\n",
    "train_dataset = SeqLabelDataset(train_X, train_Y)\n",
    "test_dataset = SeqLabelDataset(test_X, test_Y)\n",
    "\n",
    "from tqdm import tqdm, trange\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "def train(model, batch_size, epochs, learning_rate):\n",
    "    train_dataloader = DataLoader(train_dataset,\n",
    "        batch_size=batch_size, shuffle=True, \n",
    "        collate_fn=SeqLabelDataset.collate_batch)\n",
    "    optimizer = Adam(model.parameters(), lr=learning_rate)\n",
    "    model.zero_grad()\n",
    "    tr_step = []\n",
    "    tr_loss = []\n",
    "    global_step = 0\n",
    "    \n",
    "    with trange(epochs, desc='epoch', ncols=60) as pbar:\n",
    "        for epoch in pbar:\n",
    "            model.train()\n",
    "            for step, batch in enumerate(train_dataloader):\n",
    "                global_step += 1\n",
    "                loss = model(**batch)\n",
    "                loss.backward()\n",
    "                optimizer.step()\n",
    "                model.zero_grad()\n",
    "                pbar.set_description(f'epoch-{epoch}, '+\\\n",
    "                    f'loss={loss.item():.2f}')\n",
    "                \n",
    "                if epoch > 0:\n",
    "                    tr_step.append(global_step)\n",
    "                    tr_loss.append(loss.item())\n",
    "\n",
    "    # 打印损失曲线\n",
    "    plt.plot(tr_step, tr_loss, label='train loss')\n",
    "    plt.xlabel('training steps')\n",
    "    plt.ylabel('loss')\n",
    "    plt.legend()\n",
    "    plt.show()\n",
    "                \n",
    "vocab_size = len(dataset.token2id)\n",
    "hidden_size = 32\n",
    "n_tags = len(dataset.label2id)\n",
    "crf = CRF(vocab_size, hidden_size, n_tags)\n",
    "batch_size = 128\n",
    "epochs = 20\n",
    "learning_rate = 1e-2\n",
    "train(crf, batch_size, epochs, learning_rate)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "4a1e7e25",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "precision = 0.547465912014407, recall = 0.2536957558416786, f1 = 0.3467209775967413\n",
      "precision = 0.4881450488145049, recall = 0.21432945499081446, f1 = 0.29787234042553196\n"
     ]
    }
   ],
   "source": [
    "# 验证效果\n",
    "\n",
    "def evaluate(X, Y, model, batch_size):\n",
    "    dataset = SeqLabelDataset(X, Y)\n",
    "    dataloader = DataLoader(dataset, batch_size=batch_size,\\\n",
    "        collate_fn=SeqLabelDataset.collate_batch)\n",
    "    model.eval()\n",
    "    with torch.no_grad():\n",
    "        P = []\n",
    "        for batch in dataloader:\n",
    "            preds = model.decode(batch['input_ids'], batch['masks'])\n",
    "            P.extend(preds)\n",
    "\n",
    "    p, r, f = compute_metric(Y, P)\n",
    "    print(f'precision = {p}, recall = {r}, f1 = {f}')\n",
    "    \n",
    "evaluate(train_X, train_Y, crf, batch_size)\n",
    "evaluate(test_X, test_Y, crf, batch_size)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cc369a17",
   "metadata": {},
   "source": [
    "\n",
    "\n",
    "这里介绍基于双向长短期记忆-条件随机场（BiLSTM-CRF）结构的代码实现。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "990c5bc1",
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "from torch import nn\n",
    "\n",
    "# 定义模型，将长短期记忆的输出输入给条件随机场，\n",
    "# 利用条件随机场计算损失和解码\n",
    "class LSTM_CRF(nn.Module):\n",
    "    def __init__(self, vocab_size, hidden_size, num_layers,\\\n",
    "                 dropout, n_tags):\n",
    "        super().__init__()\n",
    "        self.embedding = nn.Embedding(vocab_size, hidden_size)\n",
    "        self.lstm = nn.LSTM(input_size=hidden_size,\n",
    "                            hidden_size=hidden_size,\n",
    "                            num_layers=num_layers,\n",
    "                            batch_first=False,\n",
    "                            dropout=dropout,\n",
    "                            bidirectional=True)\n",
    "        self.crf = CRFLayer(n_tags, hidden_size * 2)\n",
    "    \n",
    "    def forward(self, input_ids, masks, labels):\n",
    "        \"\"\"\n",
    "        input_ids/masks/labels: batch_size * seq_len\n",
    "        \"\"\"\n",
    "        # 将输入序列转化为词嵌入序列\n",
    "        # batch_size * seq_len * embed_size\n",
    "        embed = self.embedding(input_ids)\n",
    "        embed = torch.transpose(embed, 0, 1)\n",
    "        masks = torch.transpose(masks, 0, 1)\n",
    "        labels = torch.transpose(labels, 0, 1)\n",
    "        # 输入长短期记忆得到隐状态\n",
    "        hidden_states, _ = self.lstm(embed)\n",
    "        # 隐状态和标签、掩码输入条件随机场计算损失\n",
    "        llh = self.crf(hidden_states, labels, masks)\n",
    "        return -llh\n",
    "    \n",
    "    def decode(self, input_ids, masks):\n",
    "        # 输入长短期记忆得到隐状态\n",
    "        embed = self.embedding(input_ids)\n",
    "        embed = torch.transpose(embed, 0, 1)\n",
    "        masks = torch.transpose(masks, 0, 1)\n",
    "        hidden_states, _ = self.lstm(embed)\n",
    "        # 调用条件随机场进行解码\n",
    "        return self.crf.decode(hidden_states, masks)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "ab41a9f0",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "epoch-19, step=7, loss=39.05: 100%|█████████████| 20/20 [06:13<00:00, 18.69s/it]\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjsAAAGwCAYAAABPSaTdAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8pXeV/AAAACXBIWXMAAA9hAAAPYQGoP6dpAABylUlEQVR4nO3dd3hUZfo+8PtMyUzKZNIbCRAIBAKhCIggUpSmFOviYuW3ujZAUUHXXXe/6O6CYkGFVVfXFevirgIWRIpAAJHeeyC9kd6Tqef3x8w5mUkvE5IZ7s915Vpmzpkz58gKt8/7vO8riKIogoiIiMhDKbr6BoiIiIg6E8MOEREReTSGHSIiIvJoDDtERETk0Rh2iIiIyKMx7BAREZFHY9ghIiIij6bq6hvoDqxWK3JycqDT6SAIQlffDhEREbWCKIqoqKhAVFQUFIqm6zcMOwBycnIQExPT1bdBRERE7ZCZmYno6OgmjzPsANDpdABs/7D8/f27+G6IiIioNcrLyxETEyP/Pd4Uhh1AHrry9/dn2CEiInIzLbWgsEGZiIiIPBrDDhEREXk0hh0iIiLyaOzZISIij2exWGAymbr6NqiN1Go1lEplh6/DsENERB5LFEXk5eWhtLS0q2+F2ikgIAAREREdWgePYYeIiDyWFHTCwsLg4+PDhWPdiCiKqK6uRn5+PgAgMjKy3ddi2CEiIo9ksVjkoBMcHNzVt0Pt4O3tDQDIz89HWFhYu4e0urRB+b333sOQIUPk9W3GjBmDTZs2ycdFUcTSpUsRFRUFb29vTJw4EadPn3a6hsFgwMKFCxESEgJfX1/Mnj0bWVlZV/pRiIiom5F6dHx8fLr4TqgjpN+/jvRcdWnYiY6OxiuvvIJDhw7h0KFDuPHGG3HrrbfKgWbFihV48803sXr1ahw8eBARERGYMmUKKioq5GssWrQI69evx9q1a7Fnzx5UVlZi5syZsFgsXfVYRETUjXDoyr254vdPEEVRdMG9uExQUBBee+01/O53v0NUVBQWLVqE559/HoCtihMeHo5XX30Vjz76KMrKyhAaGorPPvsMd999N4C6fa5+/PFHTJs2rVXfWV5eDr1ej7KyMq6gTETkIWpra5GamorY2Fhotdquvh1qp+Z+H1v793e3WWfHYrFg7dq1qKqqwpgxY5Camoq8vDxMnTpVPkej0WDChAnYu3cvAODw4cMwmUxO50RFRWHw4MHyOY0xGAwoLy93+iEiIiLP1OVh5+TJk/Dz84NGo8Fjjz2G9evXIyEhAXl5eQCA8PBwp/PDw8PlY3l5efDy8kJgYGCT5zRm+fLl0Ov18g93PCciIk/Wu3dvvPXWW11+ja7S5WEnPj4ex44dw759+/D444/jwQcfxJkzZ+Tj9cfqRFFscfyupXNeeOEFlJWVyT+ZmZkdewgXqjGy14iI6Go3ceJELFq0yGXXO3jwIB555BGXXc/ddHnY8fLyQlxcHEaOHInly5dj6NChePvttxEREQEADSo0+fn5crUnIiICRqMRJSUlTZ7TGI1GI88A6047nX+4KwWJSzfj10tFXX0rRETUzYmiCLPZ3KpzQ0NDr+pZaV0eduoTRREGgwGxsbGIiIjA1q1b5WNGoxFJSUkYO3YsAGDEiBFQq9VO5+Tm5uLUqVPyOe7kYFoxzFYRJ7JKu/pWiIg8kiiKqDaau+SntfOB5s2bh6SkJLz99tsQBAGCICAtLQ07d+6EIAjYvHkzRo4cCY1Gg927d+PSpUu49dZbER4eDj8/P4waNQrbtm1zumb9IShBEPCvf/0Lt99+O3x8fNCvXz989913bfpnmZGRgVtvvRV+fn7w9/fHnDlzcPnyZfn48ePHMWnSJOh0Ovj7+2PEiBE4dOgQACA9PR2zZs1CYGAgfH19MWjQIPz4449t+v626NJFBf/4xz/i5ptvRkxMDCoqKrB27Vrs3LkTP/30EwRBwKJFi7Bs2TL069cP/fr1w7Jly+Dj44N77rkHAKDX6/HQQw/h2WefRXBwMIKCgrB48WIkJiZi8uTJXflo7VJpsCX0WpO1i++EiMgz1ZgsSPjL5i757jMvT4OPV8t/7b799tu4cOECBg8ejJdffhmArTKTlpYGAHjuuefw+uuvo0+fPggICEBWVhZuueUW/O1vf4NWq8Unn3yCWbNm4fz58+jZs2eT3/PSSy9hxYoVeO2117Bq1Srce++9SE9PR1BQUIv3KIoibrvtNvj6+iIpKQlmsxlPPPEE7r77buzcuRMAcO+992L48OF47733oFQqcezYMajVagDA/PnzYTQasWvXLvj6+uLMmTPw8/Nr8Xvbq0vDzuXLl3H//fcjNzcXer0eQ4YMwU8//YQpU6YAsP2G1tTU4IknnkBJSQlGjx6NLVu2QKfTyddYuXIlVCoV5syZg5qaGtx0001Ys2aNSzYOu9Kq7GGnxsS+HSKiq5Ver4eXlxd8fHzklg5HL7/8svz3JAAEBwdj6NCh8uu//e1vWL9+Pb777jssWLCgye+ZN28e5s6dCwBYtmwZVq1ahQMHDmD69Okt3uO2bdtw4sQJpKamypN8PvvsMwwaNAgHDx7EqFGjkJGRgSVLlmDAgAEAgH79+smfz8jIwJ133onExEQAQJ8+fVr8zo7o0rDz0UcfNXtcEAQsXboUS5cubfIcrVaLVatWYdWqVS6+uyuvrrLDsENE1Bm81Uqcebl1a7B1xne7wsiRI51eV1VV4aWXXsIPP/yAnJwcmM1m1NTUICMjo9nrDBkyRP61r68vdDqdvA9VS86ePYuYmBin2cwJCQkICAjA2bNnMWrUKDzzzDN4+OGH8dlnn2Hy5Mn4zW9+g759+wIAnnzySTz++OPYsmULJk+ejDvvvNPpflyt2/XsXM2qDLaQwxlZRESdQxAE+HipuuTHVSs5+/r6Or1esmQJvvnmG/z973/H7t27cezYMSQmJsJoNDZ7HWlIyfGfjdXaujaKpmY9O76/dOlSnD59GjNmzMD27duRkJCA9evXAwAefvhhpKSk4P7778fJkycxcuTITi1aMOx0I5UcxiIiIthmKrd226Pdu3dj3rx5uP3225GYmIiIiAi5v6ezJCQkICMjw2npljNnzqCsrAwDBw6U3+vfvz+efvppbNmyBXfccQc+/vhj+VhMTAwee+wxrFu3Ds8++yw+/PDDTrtfhp1uQhRFVBkZdoiIyDZ7av/+/UhLS0NhYWGzFZe4uDisW7cOx44dw/Hjx3HPPfe0ukLTXpMnT8aQIUNw77334siRIzhw4AAeeOABTJgwASNHjkRNTQ0WLFiAnTt3Ij09Hb/88gsOHjwoB6FFixZh8+bNSE1NxZEjR7B9+3ankORqDDvdRLXRAmlWInt2iIiubosXL4ZSqURCQgJCQ0Ob7b9ZuXIlAgMDMXbsWMyaNQvTpk3DNddc06n3JwgCNmzYgMDAQIwfPx6TJ09Gnz598NVXXwEAlEolioqK8MADD6B///6YM2cObr75Zrz00ksAbFtEzZ8/HwMHDsT06dMRHx+Pd999t/Put7ttBNoVusNGoJfLazF62c8AgFG9A/G/x9xvnSAiou6EG4F6Bo/aCPRqJ/XrABzGIiIiciWGnW6iyjHscDYWERGRyzDsdBOVtXVhhysoExERuQ7DTjfhOIzFBmUiItdha6p7c8XvH8NONyFNOwfYs0NE5ArSonnV1dVdfCfUEdLvX/1FENuiS7eLoDqVhrqAU2OyNLk6JRERtY5SqURAQIC8BYKPjw//XHUjoiiiuroa+fn5CAgI6NCelww73YRjz44oAgazFVoX7aNCRHS1kjbSbO2eT9T9BAQENLohalsw7HQTjrOxAFvfDsMOEVHHCIKAyMhIhIWFwWQydfXtUBup1eoOVXQkDDvdRGWDsMMZWURErqJUKl3ylya5JzYodxP1ww6blImIiFyDYaebqD+MxYUFiYiIXINhp5tgZYeIiKhzMOx0Ew17dhh2iIiIXIFhp5vgMBYREVHnYNjpJqrsiwp626eb15oZdoiIiFyBYaebqKi1rf8QovMCwMoOERGRqzDsdAOiKKLKHm5C/DQA2LNDRETkKgw73YDBbIXFatvVNdQedjgbi4iIyDUYdroBx5lYwX7SMBZXUCYiInIFhp1uQNoE1NdLCW+1bQcPNigTERG5BsNONyBVdvy0Knh72X5L2KBMRETkGgw73YC0xo6vRlU39Zw9O0RERC7BsNMFzBYrzuWVQxRtTclyZUejgtYedtigTERE5BoMO13gtS3nMf2t3dh4MheAc9jx9rKHHQ5jERERuQTDzhVmtljx9aEsAMCR9FIAdasn+2pU0KpY2SEiInIlhp0r7EBqMYqqjACA7NJqAEClwbZ6smNlx2Di1HMiIiJXYNi5wn6wD10BQHZpDQCg0l7Z8XNoUGZlh4iIyDUYdq4gs8WKzafy5NfZJbaw4zgbiw3KRERErsWwcwVJQ1g6jW3hwJJqE6oMZjns+GmUbFAmIiJyMYadK0gawpoxJBL+WlvgyS6tQYXjbCyus0NERORSDDtXiOMQ1i2JkegR6APANpTlPIxl+y1h2CEiInINhp0rRBrCCvRRY0zfYPQI8AYAZJXWOAxjOTcoS4sOEhERUfupuvoGPJ0oithwLBuvbDoHAJg2KAJqpQLRgbawk11Sg4raur2xtPaeHasIGC1WaOzr7hAREVH7MOx0ohNZpVj63WkcySgFAPQM8sETE+MAQK7sZJfWoMrYcG8sAKg1MuwQERF1FMNOJ1rx03kcySiFj5cS8yfF4aFxsfLU8h72yk5WSbW8grKfRgW1UgGVQoDZKqLGZIEe6i67fyIiIk/AsNOJ/jwzAf9MuoTnpg9AhF7rdEyu7JTUoLK2rmcHALRqJSoNZjYpExERuQDDTieKj9DhzbuHNXpMquzkVxjk93zrhR0uLEhERNRxnI3VRYJ9veRp5hJfe3Oyt5ftfYYdIiKijmPY6SKCIMhDWQCgVSugUtp+O+SFBbmKMhERUYcx7HQhaWFBAPDT1DUiczNQIiIi12HY6UKOlR0/Td0Uc24GSkRE5DoMO11IWlgQqGtOBurCTq3JesXviYiIyNMw7HQh58pOXdjhMBYREZHrMOx0oR6BTYQdLzYoExERuQrDThdyrOw0NozFyg4REVHHMex0oXB/LVQKAYBtE1AJh7GIiIhch2GnCykVgryNhJ9TZcf228LtIoiIiDqOYaeLSUNZvl4NKzsMO0RERB3XpWFn+fLlGDVqFHQ6HcLCwnDbbbfh/PnzTufMmzcPgiA4/Vx33XVO5xgMBixcuBAhISHw9fXF7NmzkZWVdSUfpd0SovwBOE9DlxqUa9igTERE1GFdGnaSkpIwf/587Nu3D1u3boXZbMbUqVNRVVXldN706dORm5sr//z4449OxxctWoT169dj7dq12LNnDyorKzFz5kxYLN0/LDw7NR6fPzQatw6Lkt9jgzIREZHrdOmu5z/99JPT648//hhhYWE4fPgwxo8fL7+v0WgQERHR6DXKysrw0Ucf4bPPPsPkyZMBAJ9//jliYmKwbds2TJs2rcFnDAYDDIa63cbLy8td8Tjt4qdRYVy/EKf36hqUuaggERFRR3Wrnp2ysjIAQFBQkNP7O3fuRFhYGPr374/f//73yM/Pl48dPnwYJpMJU6dOld+LiorC4MGDsXfv3ka/Z/ny5dDr9fJPTExMJzxN+3GdHSIiItfpNmFHFEU888wzGDduHAYPHiy/f/PNN+OLL77A9u3b8cYbb+DgwYO48cYb5cpMXl4evLy8EBgY6HS98PBw5OXlNfpdL7zwAsrKyuSfzMzMznuwdpBnY5kZdoiIiDqqS4exHC1YsAAnTpzAnj17nN6/++675V8PHjwYI0eORK9evbBx40bccccdTV5PFEUIgtDoMY1GA41G45ob7wRyzw4rO0RERB3WLSo7CxcuxHfffYcdO3YgOjq62XMjIyPRq1cvJCcnAwAiIiJgNBpRUlLidF5+fj7Cw8M77Z47ExcVJCIicp0uDTuiKGLBggVYt24dtm/fjtjY2BY/U1RUhMzMTERGRgIARowYAbVaja1bt8rn5Obm4tSpUxg7dmyn3Xtnknt2GHaIiIg6rEuHsebPn48vv/wS3377LXQ6ndxjo9fr4e3tjcrKSixduhR33nknIiMjkZaWhj/+8Y8ICQnB7bffLp/70EMP4dlnn0VwcDCCgoKwePFiJCYmyrOz3I03h7GIiIhcpkvDznvvvQcAmDhxotP7H3/8MebNmwelUomTJ0/i008/RWlpKSIjIzFp0iR89dVX0Ol08vkrV66ESqXCnDlzUFNTg5tuuglr1qyBUqm8ko/jMlLPTq3Z2mzvEREREbVMEEVR7Oqb6Grl5eXQ6/UoKyuDv79/V98OympMGPrSFgDAhb/dDC9Vt2itIiIi6lZa+/c3/xbthqRhLIBNykRERB3FsNMNqZUClArb0BWblImIiDqGYacbEgSBTcpEREQuwrDTTdU1KTPsEBERdQTDTjclbRnByg4REVHHMOx0U1xFmYiIyDUYdroprqJMRETkGgw73VTdZqDWLr4TIiIi98aw001xGIuIiMg1GHa6KSnsnMou6+I7ISIicm8MO93U1EHhAIA1e9Pw7s6LXXw3RERE7othp5u645poLJkWDwBY8dN5fLDrUhffERERkXti2OnG5k+Kw9OT+wMAlv14Dr9eKuriOyIiInI/DDvd3FOT+2HywDAAwMns0q69GSIiIjfEsOMGegb5AgCKqoxdfCdERETuh2HHDQT7eQEAShh2iIiI2oxhxw0E+tjCTnGVqYvvhIiIyP0w7LiBIF81AKCkmpUdIiKitmLYcQNSZYfDWERERG3HsOMGgnztw1is7BAREbUZw44bCLSHnbIaE8wWbgxKRETUFgw7biDAWw1BAEQRKK1hkzIREVFbMOy4AZVSAb23vUmZfTtERERtwrDjJoLk6ecMO0RERG3BsOMmpL4dTj8nIiJqG4YdN8GFBYmIiNqHYcdNcGFBIiKi9mHYcRPSMBZ7doiIiNqGYcdNBDPsEBERtQvDjpsI5GwsIiKidmHYcRNBnI1FRETULgw7boI9O0RERO3DsOMmgrjzORERUbsw7LgJqbJTZbSg1mTp4rshIiJyHww7bsJfq4JKIQAASqu5sCAREVFrMey4CUEQ5OpOUZWhi++GiIjIfTDsuJG6vh1WdoiIiFqLYceNBNq3jCjm9HMiIqJWY9hxI/JaO5yRRURE1GoMO26EqygTERG1HcOOG+EqykRERG3HsONGpMpOESs7RERErcaw40aC/dizQ0RE1FYMO26EPTtERERtx7DjRtizQ0RE1HYMO24k0LduUUFRFLv4boiIiNwDw44bkVZQNlqsqDJyM1AiIqLWYNhxI95eSmjVtt8yNikTERG1DsOOmwni9HMiIqI2YdhxM0Gcfk5ERNQmDDtuhtPPiYiI2oZhx81w+jkREVHbdGnYWb58OUaNGgWdToewsDDcdtttOH/+vNM5oihi6dKliIqKgre3NyZOnIjTp087nWMwGLBw4UKEhITA19cXs2fPRlZW1pV8lCuGlR0iIqK26dKwk5SUhPnz52Pfvn3YunUrzGYzpk6diqqqKvmcFStW4M0338Tq1atx8OBBREREYMqUKaioqJDPWbRoEdavX4+1a9diz549qKysxMyZM2GxeN70bFZ2iIiI2kYQu9HqdAUFBQgLC0NSUhLGjx8PURQRFRWFRYsW4fnnnwdgq+KEh4fj1VdfxaOPPoqysjKEhobis88+w9133w0AyMnJQUxMDH788UdMmzatxe8tLy+HXq9HWVkZ/P39O/UZO+qzfen484ZTmDYoHP+8f2RX3w4REVGXae3f392qZ6esrAwAEBQUBABITU1FXl4epk6dKp+j0WgwYcIE7N27FwBw+PBhmEwmp3OioqIwePBg+Zz6DAYDysvLnX7cRRCHsYiIiNqk24QdURTxzDPPYNy4cRg8eDAAIC8vDwAQHh7udG54eLh8LC8vD15eXggMDGzynPqWL18OvV4v/8TExLj6cTqNNIzFsENERNQ63SbsLFiwACdOnMB//vOfBscEQXB6LYpig/fqa+6cF154AWVlZfJPZmZm+2/8Cqvr2TF16Do/nMjBY58dRkVtx65DRETU3XWLsLNw4UJ899132LFjB6Kjo+X3IyIiAKBBhSY/P1+u9kRERMBoNKKkpKTJc+rTaDTw9/d3+nEXgb5qAEBptREWa1271d6LhVj8v+Moq2ldeFn180X8dDoPe5ILO+U+iYiIuosuDTuiKGLBggVYt24dtm/fjtjYWKfjsbGxiIiIwNatW+X3jEYjkpKSMHbsWADAiBEjoFarnc7Jzc3FqVOn5HM8iTT13CoC5Q7B5h87L+Lrw1nYdDK3xWtYrCJSC20z3jpaISIiIuruVF355fPnz8eXX36Jb7/9FjqdTq7g6PV6eHt7QxAELFq0CMuWLUO/fv3Qr18/LFu2DD4+Prjnnnvkcx966CE8++yzCA4ORlBQEBYvXozExERMnjy5Kx+vU6iVCui0KlTUmlFcbUSgfVgrt6wWAJBZUt3iNbJKqmG0WAEApTXs/SEiIs/WpWHnvffeAwBMnDjR6f2PP/4Y8+bNAwA899xzqKmpwRNPPIGSkhKMHj0aW7ZsgU6nk89fuXIlVCoV5syZg5qaGtx0001Ys2YNlErllXqUKyrI1wsVtWbb/lihtvfyyw0AgOySmhY/n1JQt45Ra4e9iIiI3FWXhp3WLPEjCAKWLl2KpUuXNnmOVqvFqlWrsGrVKhfeXfcV5OuF9KJqeefzSoMZlQYzACCrFWHnUkGl/OsyDmMREZGH6xYNytQ20lo70s7n+eW18rHs0taEHVZ2iIjo6sGw44akPp1i+5YRl+1DWACQV14Lo9na7OdTHCo7pazsEBGRh2PYcUPyWjtSZaeirrIjikBeWW2jn5OwskNERFcThh03VLfzuS2oXC53DjdZpU3PyCqvNaGwsq4SxLBDRESejmHHDQXZFxaUdj7PdxjGAppvUpZmYkmLS5dy93QiIvJwDDtuKLDeZqCXK2xhR2EPMM1NP7+Ub+vXiQ+3Td2vMlpgsjTf40NEROTOGHbcULBfvbBjH8bqbw8wzVZ2Cm1hZ3jPAPk9DmUREZEnY9hxQ4FNTD0f0cu283t2Mz07l/Jtw1hxYTrotLZllhh2iIjIkzHsuCFpNlaFwQyj2SpPPb+mpxR2Wq7s9A31RYCPtKkoww4REXkuhh035K9Vy/05GcXVqDFZAADX2Cs7uaW1TjuiSyxWEWmFtqpP31A/6L1tYaeM+2MREZEHY9hxQwqFIA9lncsrBwD4a1XoGeQDlUKA2So2mI4O1G0A6qVSICrAGwHetmtwGIuIiDwZw46bklZRPpdbAQAI99dCqRAQFeANoPEmZWnaeZ8QXygVglzZcRzGemvbBdywYjve2nYBRZWGBtcgIiJyNww7biqoXmUn3F8LAOhhDzuNNSlLG4D2CfUFAOh9pGGsurCz4Wg2Motr8Na2ZIx5ZTv+8u2pFrefICIi6s4YdtyU1KR81l7ZCfPXAAB6BNrDTiOVnUtyZccPABBQr7IjiiJy7VtN9Avzg9Fsxae/puNgWnFnPQYREVGnY9hxU9IwljTzSqrsRAc2PYyVWlivsmMPO+X2yk5xlREGexXnhyfH4drYIABw2l6CiIjI3TDsuClpywhJuM5e2ZGHsRqGHWlbCamvR556bg87UlUnxE8DjUqJED/n9XyIiIjcEcOOm5JmY0nqKjs+ABqv7OTbt5UItQejugZlW5iRwk5UgNbpO0q4Dg8REbkxhh03JfXsSKSeHWkYK7u0BlaHtXaqjWZUGswAHMOO89Tz3DJbQIrUO4cdbhZKRETujGHHTQXWDzs6W0CJ0GuhEACj2YrCqrpem8IKW2DRqhXQaWzbRATUm42VU2qr7ETqnYe5WNkhIiJ3xrDjpoJ8Gq/sqJUKRNiHtByHsvIrbEEmVKeBINiWX65bQdkEURSR10Rlp4SVHSIicmMMO27KcRgr0EcNjUopv5b6dhynnxfY+3WkChBQV7kxWURUGy3IsffsRNobmAN9pcoOww4REbkvhh035Rh2pOZkidRgnOMwI6vAPn081E8jv+etVkKttFV5SmtMcs9OlL2yEyDvrs5hLCIicl8MO27Kx0sJL5Xtty+sXtiRFhZ0DDvStHOpORkABEGQm5RLqozIs1d2ItigTEREHoRhx00JgiD37YQ7BBigbh2d7NK6zUDrhrGcz5WGslIKq2CyiBCEukpRoP1YldHCLSOIiMhtMey4MWlGVsNhrIYLC8rDWPXCjtSkfC7XtsdWmE4DtdL2fwt/rRoK2ygXqztEROS2GHbcmLSKsjQTSyKtouw0jOUwG8uRtD/WWXvYkaadA4BCIdT17XD6ORERuSlVV98Atd+9o3vBZBYxeWC40/vS1PGyGhMqDWb4aVSNzsYCHCo7eRVOn5UE+KhRXGXkjCwiInJbDDtu7JbESNySGNngfZ1WDX+tCuW1ZuSW1qBvqB8KK21hpcEwlr0vR9oqwrGyA0hNylUcxiIiIrfVrmGsTz75BBs3bpRfP/fccwgICMDYsWORnp7uspuj9nPs2ymuNsJitTUfB/s5L0YY4O1V73POlZ1ArqJMRERurl1hZ9myZfD2tv1l+uuvv2L16tVYsWIFQkJC8PTTT7v0Bql96vp2auUhrCAfL7n5WKL3di7uRTQYxuIqykRE5N7aNYyVmZmJuLg4AMCGDRtw11134ZFHHsH111+PiRMnuvL+qJ0c19qRNgetP4QF1IUZScNhLHtlp4phh4iI3FO7Kjt+fn4oKioCAGzZsgWTJ08GAGi1WtTU1DT3UbpCHIex8isan3YO1DUo132uqcoOh7GIiMg9tauyM2XKFDz88MMYPnw4Lly4gBkzZgAATp8+jd69e7vy/qidHMNOQXNhx6cu7CgVQoPZWlxFmYiI3F27Kjv/+Mc/MGbMGBQUFOCbb75BcHAwAODw4cOYO3euS2+Q2qeHw/5Y0ho79YMM4FzZCdNpoJRWEbQL8mWDMhERubd2VXYCAgKwevXqBu+/9NJLHb4hcg2pspNXVovL5Y0vKAjULSoINFxjB2CDMhERub92VXZ++ukn7NmzR379j3/8A8OGDcM999yDkpISl90ctV+YTgulQoDZKuJMjm115JZ6diIDvBscrxvGaryyk11ag/0pRa64ZSIiok7RrrCzZMkSlJfb/gI9efIknn32Wdxyyy1ISUnBM88849IbpPZRKgRE2PfMSiuqBtBwE1AAUCkV8NPYCnxRjVR2pNlYpdVGWK2i07FKgxl3vrsXd3+wDyezylx6/0RERK7SrrCTmpqKhIQEAMA333yDmTNnYtmyZXj33XexadMml94gtV+PepWaxio7QF11J0LfsLIjDWNZRaCi1ux07J2fk5FnHyL7+dzlDt8vERFRZ2hX2PHy8kJ1ta1asG3bNkydOhUAEBQUJFd8qOvVn0beWGUHAILsu6f3CGhY2fFSKeDrpQTg3LeTfLkC/96TKr/enVzY4fslIiLqDO0KO+PGjcMzzzyDv/71rzhw4IA89fzChQuIjo526Q1S+0kLCwKAVl03XFXfkzf1w10jojExPqzR4/WblEVRxF++PQ2zVcQ1PQMAAMcyS1FWwxlbRETU/bQr7KxevRoqlQpff/013nvvPfTo0QMAsGnTJkyfPt2lN0jtF+UwjBWq00AQhEbPm5IQjtd/MxRatbLR44Hy9HNb2PnhRC5+TSmCRqXA278djj6hvrBYRfx6iY3KRETU/bRr6nnPnj3xww8/NHh/5cqVHb4hch3HsNPYGjutJc3IKqmyVW7e+TkZAPDExDjEBPlgfL9QpBRUYXdyAaYPjujAHRMREbleu8IOAFgsFmzYsAFnz56FIAgYOHAgbr31ViiVjVcH6MpzbFAO9Wu8X6c1Ah2GsS6X1yI5vxKCAMwb2xsAcEO/EKzZm8a+HSIi6pbaFXYuXryIW265BdnZ2YiPj4coirhw4QJiYmKwceNG9O3b19X3Se3guEhgmH9Hwo40/dyEffY1dQZF+ctbTVzXJxhqpYCM4mqkF1WhV7BvB+6aiIjItdrVs/Pkk0+ib9++yMzMxJEjR3D06FFkZGQgNjYWTz75pKvvkdpJp1XDX2vLsx2p7Dg2KEt9OWP6BMvHfTUqXNMzEACwq5Hqzh++OYGb3tiJ7FJuEktERFdeu8JOUlISVqxYgaCgIPm94OBgvPLKK0hKSnLZzVHHSX07Ta2x0xqOlZ1f7ZWdMX2Dnc4Z3z8UALDrQoHT+6IoYv3RbFwqqMKz/z0GS72FCYmIiDpbu8KORqNBRUVFg/crKyvh5eXV4Zsi15k6KAI6rQqjYoNaPrkJgfZ1eM7kliO9qBoKARjZ2/l64/vZws6vl4pgsljl9wsrjTCYba/3pRTjX7tT2n0fRERE7dGusDNz5kw88sgj2L9/P0RRhCiK2LdvHx577DHMnj3b1fdIHfDMlP44/pep6Bvq1+5rSMNYqYVVAIDEHnr4a9VO5wyK8keAjxqVBrO8FxcAZJXYFp+UNlN/fct5nM7h1hJERHTltCvsvPPOO+jbty/GjBkDrVYLrVaLsWPHIi4uDm+99ZaLb5E6SqFofH2d1pKGsSTX1RvCkr6jf7gOAJBWVCW/n1Vi69MZ0SsQUxPCYbKIWLT2mFP1h4iIqDO1azZWQEAAvv32W1y8eBFnz56FKIpISEhAXFycq++PugFp6rnEsTnZUe9gHxxILUZaYbX8nhR2YgJ98OLMBOxL2YHk/EqcyCrDiF6BnXfTREREdq0OOy3tZr5z507512+++Wa7b4i6H6lnB7Dtpj6qd+P9P9KU8/Rix8qOLfj0CPRGkK8XBvfQY++lIqQUVDqFnYc/OYhLBVX4fuG4Jre1ICIiao9W/61y9OjRVp3X1JYE5L58vZRQKwWYLCKGROvh20QY6RXsAwBIL6qr7EjTzaPt+3T1CfW1hZ3CukBUUWvCtrP5AGwNzlMSwjvlOYiI6OrU6rCzY8eOzrwP6sYEQUCAjxcKKgxNDmEBQK8ge2WnkZ6d6EBbEOoTYmuUTimolM+5mF/36wOpDDtERORa7WpQdpVdu3Zh1qxZiIqKgiAI2LBhg9PxefPmQRAEp5/rrrvO6RyDwYCFCxciJCQEvr6+mD17NrKysq7gU1wdpK0nbrBPMW9MT3tlp7DSiEqDGaIoysNYUmWnb5gUduoCUfJlx7BT7NobJyKiq16Xhp2qqioMHToUq1evbvKc6dOnIzc3V/758ccfnY4vWrQI69evx9q1a7Fnzx5UVlZi5syZsFgsnX37V5XXfzME78wdjuv6NL1ej95bjSB7f096URWKqoyoNVkhCECk3j6MFWKr/qQVVcFsn5F14XLdmk2ncspRaTB31mMQEdFVqEs7QW+++WbcfPPNzZ6j0WgQEdH4TtplZWX46KOP8Nlnn2Hy5MkAgM8//xwxMTHYtm0bpk2b5vJ7vlrFhekQF6Zr8byeQT4orjIio6gaZottteRwnRZeKluu7hHgDY1KAYPZiqySGvQO8cUFh2Esi1XE4fQSTOjfdAWJiIioLbq0stMaO3fuRFhYGPr374/f//73yM/Pl48dPnwYJpMJU6dOld+LiorC4MGDsXfv3iavaTAYUF5e7vRDrtHbPpSVVlTt0K9Tt/u6QiEg1l7dSSm0hZxke2VHqvocSC26YvdLRESer1uHnZtvvhlffPEFtm/fjjfeeAMHDx7EjTfeCIPBAADIy8uDl5cXAgOd12sJDw9HXl5ek9ddvnw59Hq9/BMTE9Opz3E16Wmffp5RXNWgX0cireacUlCF8loTcstqAQD3jO4JgH07RETkWt067Nx9992YMWMGBg8ejFmzZmHTpk24cOECNm7c2OznRFFsdgr8Cy+8gLKyMvknMzPT1bd+1ZIrO4XVDWZiSfqE2gLRpYIquTk53F8jz8I6nlmGWhN7roiIyDW6ddipLzIyEr169UJycjIAICIiAkajESUlJU7n5efnIzy86enLGo0G/v7+Tj/kGnVr7VTJa+z0qFfZqQs7lfIQVv9wHXoG+SDcXwOjxYqjGaVX7qaJiMijuVXYKSoqQmZmJiIjIwEAI0aMgFqtxtatW+VzcnNzcerUKYwdO7arbvOqJq2inFtei0v2tXTqD2PVrbVThQv2yk6/MB0EQcC1sbZ1fDiURURErtKls7EqKytx8eJF+XVqaiqOHTuGoKAgBAUFYenSpbjzzjsRGRmJtLQ0/PGPf0RISAhuv/12AIBer8dDDz2EZ599FsHBwQgKCsLixYuRmJgoz86iKyvY1wu+XkpUGS3ySspNDWMVVhpwOMNWlesfbgtAo2OD8P3xHBxIKwLQ78rdOBEReawuDTuHDh3CpEmT5NfS/lsPPvgg3nvvPZw8eRKffvopSktLERkZiUmTJuGrr76CTlc3BXrlypVQqVSYM2cOampqcNNNN2HNmjVQKpVX/HnIttpyr2BfnMmtm+EWFaB1OkenVSNMp0F+hQHHM0sBAP3sO6aPjrWt43M4vQRGs1West4aZosVT3xxBApBwHv3XdNk39Yrm87BW63EU5MZpoiIrgZdGnYmTpwIURSbPL558+YWr6HVarFq1SqsWrXKlbdGHdAr2EcOO2E6DTSqhsGzT6gv8isM8ut+9spOXJgfAn3UKKk24UxuOYbFBDT6HaIowiraNiaVfHMkC1vOXAYA5FcYEO6vbfC5ggoD3k+6BACYN7Y39D7q9j0kERG5Dbfq2SH3IPXtAA37dSR97NPPASBSr4W/1hY6BEHA8J62pQSOZpQ0+llRFHHX+7/ipjd2Is8+bb3WZMHKrcnyOfnlhkY/e7m8Vv71xYKKRs8hIiLPwrBDLidNPwca9utIpAUEgbohLMlwezXnmH2Iq77TOeU4nF6CtKJqPPr5YdSaLFizNw15DkGmoLK20c8WOFSTHPfkIiIiz9Wlw1jkmXo6hZ3GKzt9HSo7/cP8nI4N6xkAAE1OP//5bN0q2sczS/Hs/45j94UCAICPlxLVRkuTlZ38CofKTj7DDhHR1YCVHXK53k7DWI1XdpzCTr3KztCYAAgCkFFcjcLKhqFl+zlbX84dw3tAqRCw8UQuymvNGBChw4xE27IEjv1AjhxD0MUChh0ioqsBww65XIS/w8afTVR2egR6y+dIzckSf60acfYwdKxedSe/ohbHs8oAAH+4ZQD+dMtA+dhz0+MRobc1JRc0EXYuO1R2OIxFRHR14DAWuZxCIWBi/1AcSCvGkB76Rs9RKgQsntof5/MqMSQ6oMHxYTEBSM6vxNHMEkxOqFsNe+c523DVkGg9wnRa/L/re8NiFWG0WDEpPkzeosJxuMqRY2Unu7QG1UYzfLz4rwERkSfjn/LUKf55/wgYLdZGp51LHhnft8ljw3sG4n+Hsxo0Kf9sH8K6cUAYANvsrd+P7yMfD9NpADRd2ak/vJVSUIXBTQQyIiLyDBzGok4hCEKzQaclw+1Nysczy2Cx2tZiMpgt2J1cCAC4aUDje5+F6mzDWE317EghyMfLdm/J+Zx+TkTk6Rh2qFvqH66Dj5cSlQazPGvqQGoxqo0WhOk0GBTV+OatjpWd+gtWiqIoh51r7Ss1c0YWEZHnY9ihbkmpEDAk2ja8JC0uKE05v3FAGBSKxreCCLWHHYPZivJas9Ox0moTjBYrAGBMH9uGoww7RESej2GHuq26lZRLcTyzFD+cyAVQ16/TGK1aCZ3W1opWUK9JWRraCvBRI8FeGUpm2CEi8nhsUKZuS1pJ+YcTOfjf4UxYRaBHgDfG9Qtp9nNhOg0qas3IrzAgLqxuDR9phlaYToM4+0KG6UXVbd5wlIiI3Av/hKduS1pJucpogVUEbh0WhW8XXN/iVPEwXeNr7UjTzsN0WkT4a+GnUcFiFZFeVOX6myciom6DlR3qtsJ0WkwbFI6Ugiq8ODMBE/qHtupzoU1MP5eGscJ0GgiCgL5hfjieWYrk/MoG+3MREZHnYNihbu2f949s82ekGVn1p59Lw1ih/rbjcaG2sMMmZSIiz8ZhLPI4UmUnv7zxBmVpmEvq22HYISLybAw75HHC7JWbgnqbiBaU1w1jAUA/hh0ioqsCww55HKly47gPFuA8Gwuoq+xcKqiUV2l2hcPpJcgsrnbZ9YiIqGMYdsjjyA3KlU00KPvbwlBMkA98vJQwmK24VOCa6k56URV+8/5e/L81B11yPSIi6jiGHfI4UuWmtNoEg9kCAKg0mFFttDgdVyoEeduJk1llLvnuQ2klsIpAamGVS6tFRETUfgw75HH03mp4KW3/1y6sNAKoa1b29VLCV1M3CVHa8fxUjmvCzsls23UsVhHFVUaXXJOIiDqGYYc8jiAIDWZk1R/CkiRKYSfbtWHH9p21zZxJRERXCsMOeaTQemvtSP8rvS+Rws7pnPIODzuZLVacdqgQ1V/UkIiIugbDDnmk+qsoSxWe8HqVnT6hfvDxUqLaaEFqYcealC8VVKHWZJVf11/UkIiIugbDDnmk+qsoF1Q4r7EjUSoEJETam5QdhqDSi6qQV9a2YagTWaVOr1nZISLqHhh2yCM1qOw0EXaAuiblk1nlAICc0hpMf2s3pr21C9mlNU7n1posTQ53SX0/gmB7XX8FZyIi6hoMO+SR6nY+lxqU7QsK+jcddqSw8sX+dNSYLCirMeGZr47J4eZ4ZinGvrIdcz/Y1+h3SpWh4TEB9u9kZYeIqDtg2CGPFNagZ8d5XyxHdU3KZag1WfCfA5kAbBWa/anFeD/pEk5kleK+j/ajuMqIA2nFqDaana5htlhxJtdWGbppYLjTdxMRUddi2CGP5Dgbq6LWhMvlzltFOOob6gutWoEqowWrt19EcZURkXotlt+eCABYufUC7v3XflTU1gWczGLn4a2LBZWoNVnhp1FhVO8g+buJiKjrMeyQR5KGq3LLapG4dAvK7UGlscqOSqmQm5T/uesSAOC+63rh7lExmDEkEmariIpaM0b2CpQ3D00vqnK6xgn7CsyDovwR7i8FrVqIIldRJiLqagw75JFC/TSI0tcFmyBfL9wzuif0PupGz5eGskwWEV5KBe4eFQNBELDstkRc2zsIUxPCseZ316J/hA4AkFFvo0+p32dItF4OVLUmKyoMzsNdRER05alaPoXI/aiUCvz09HjkldWiR4C30xYRjZGalAFg5pBIhPjZqjN6HzX++9gY+VjPIB8ADcOOVNkZ3EMPby8ldBoVKgxm5Jcb4K9tPGAREdGVwcoOeSx/rRr9w3UtBh0ASIyuCzsPjO3d5Hm9Ggk7JosVZ+3NyUOiAwAAof7ODdJtZTRb8cf1J/HTqdx2fZ6IiOqwskMEoH+YDr8dFQNfjQrD7FPHG9Mz2B52iurCzsX8ShjMVug0KjkMhfppkFJQ1e79sX65WIgv92dgT3Ihpg+ObNc1iIjIhmGHCIBCIeCVO4e0eJ40jJVZUg2LVYRSIcj9OglR/lAobCsKShuOtreyc6mgUv6eWpMFWrWyXdchIiIOYxG1SaTeG2qlAJNFRJ59OvvpHNsQ1qCouqGw+ttVNMdgtjR4L80+20sUgdTCqgbHiYio9VjZIWoDpUJAdKAPUgurkF5UhR4B3jhjDzuDe/jL58lhp5EtI0wWK/647iROZJUht6wG5bVm3DUiGq//Zqh8TrrDMNmlgkoMjPRvcB0iImodVnaI2kgeyiquhtUq4nSOtMaOQ2VHalCubFjZOZhajP8dzsL5yxXy+j8/nsyF1WHPLcdqzsX8ju3GTkR0tWPYIWqjXvYm5fSiaqQXV6PKaIFGpUDfUF/5nFA/W8+OtE2FoxR7kBnVOxCbF42Hl0qBaqMFWSW2VZkNZgtyHDYgvVTAYSwioo5g2CFqI6myk15cLVd1BkTooFLW/esU5t90z06aPewk9ghAfIQOcaG2VZnP5dmGwzKLq+G4sfolVnaIiDqEYYeojRyHsU5l25uTHRYlBOp6dspqTKg1OTcgS83HsSG26wywr8p8Pq8CAJBaaOvX0XvbFiNMKax0GuIiIqK2YdghaqOeDsNYdf06zg3Eem81vOyVnvrTz6V+nN4htmEvaQuK85dtYUeq/IztGwy1UkCtyYrsUueNR4mIqPUYdojaSKrslNWYcCS9BIBzczIACIIg77zu2KRssYryjum9g21hJ75eZUeq/PQN9ZPPkdbdISKitmPYIWojHy+VHGSqjBYoFYI8FOUoVJ5+Xhd2ckprYLRY4aVUICrAG0DdMFZKYRUMZoscdnqH+CLOvss6m5SJiNqPYYeoHaTqDgDEhfo1usKx1LdT4LBlhDQTq2ewD5T21ZYj/LXQaVWwWEVcyq9Cmr1nJzbEB31DpbDDyg4RUXsx7BC1Qy+HsFO/X0fS2IwsqR9HGp4CbENeUnXnZHYpcsrqhrn6htnO41o7RETtx7BD1A4xDmEnoamwo2u41o7UnNzHYU0eoK5vZ+uZyxBFQKdVIcjXC3Gh9iEuVnaIiNqNYYeoHaSFBQFgcL1p55LGGpTlfpzgemEn3BZqdiUXAgBiQ3whCIIcigorjSitNrro7omIri4MO0Tt4Bh2mq7sSMNYdT078jBWiI/TufERtmsYzVb79W0hx1ejQqTeViFikzIRUfsw7BC1w8BIf0QHeuPGAWHw16obPSfCHlLSC6thNFthsliRad8SIjak8cqOJNYhTMkzsti3Q0TULtz1nKgdfLxUSFoyCfYJVY0aEOGPMJ0G+RUG7Dyfj37hOlisIrRqBcLt/TwSvY8aEf5a5Nl3Se/tEIb6hvphd3IhZ2QREbUTKztE7aRUCBCEptOOUiFg9tAoAMC3x3KcZmIpGklJ8Q5r9TiHHS4sSETUEQw7RJ3otuE9AADbzl7GyWzb1hL1m5MljgsTxjqc098+xHU8qwyiyD2yiIjaqkvDzq5duzBr1ixERUVBEARs2LDB6bgoili6dCmioqLg7e2NiRMn4vTp007nGAwGLFy4ECEhIfD19cXs2bORlZV1BZ+CqGmDovwRF+YHg9mKT/amAXCu2jiSQo2/VoUAn7o+oGE9A+CtVqKgwoCzuRWdfs9ERJ6mS8NOVVUVhg4ditWrVzd6fMWKFXjzzTexevVqHDx4EBEREZgyZQoqKur+wF+0aBHWr1+PtWvXYs+ePaisrMTMmTNhsVgavSbRlSQIAm4bZhvKKqqyTR3v00TYGdM3GD5eSkyID3MaHtOolBjTNxgAsCu5oNHPGs1WvLHlPA6nF7vy9omIPIIgdpO6uCAIWL9+PW677TYAtqpOVFQUFi1ahOeffx6ArYoTHh6OV199FY8++ijKysoQGhqKzz77DHfffTcAICcnBzExMfjxxx8xbdq0Rr/LYDDAYKhb+6S8vBwxMTEoKyuDv3/j04iJ2iuzuBo3rNghv/7vo2NwbWxQo+fWGC3QqhUNeoHW/JKKpd+fwdi+wfjy99c1+Nz/DmViydcnEBfmh23PTHDtAxARdVPl5eXQ6/Ut/v3dbXt2UlNTkZeXh6lTp8rvaTQaTJgwAXv37gUAHD58GCaTyemcqKgoDB48WD6nMcuXL4der5d/YmJiOu9B6KoXE+SDkb0C5df119hx5O2lbLTpeXz/UADAobQSVBvNDY7/eqkIgG1biezSmo7eMhGRR+m2YScvLw8AEB4e7vR+eHi4fCwvLw9eXl4IDAxs8pzGvPDCCygrK5N/MjMzXXz3RM5utTcq+3opEeqnafPnY0N8ER3oDaPFin0pRU7HRFHE3kt17+260PhQFxHR1arbhh1J/f/KFUWx2em+rTlHo9HA39/f6YeoM906LApj+gTjoXGxLf7/tzGCIMjVnV0XCp2OpRVVy+vzAMDuJvp6iIiuVt027ERERABAgwpNfn6+XO2JiIiA0WhESUlJk+cQdQf+WjX+88h1eGZqfLuvMb6fFHacw4w0hBXk6wUA2JNcCLPF2u7vISLyNN027MTGxiIiIgJbt26V3zMajUhKSsLYsWMBACNGjIBarXY6Jzc3F6dOnZLPIfIUY+OCoVQISCmsQmZxtfz+3ku2Ss99o3tC761Gea0Zx7PKuuo2iYi6nS4NO5WVlTh27BiOHTsGwNaUfOzYMWRkZEAQBCxatAjLli3D+vXrcerUKcybNw8+Pj645557AAB6vR4PPfQQnn32Wfz88884evQo7rvvPiQmJmLy5Mld+GREruevVeOangEA6qagi6KIfSm26ebXx4VgXFyI7Xgb+nZEUcTF/Ep5E1IiIk/TpWHn0KFDGD58OIYPHw4AeOaZZzB8+HD85S9/AQA899xzWLRoEZ544gmMHDkS2dnZ2LJlC3S6upVmV65cidtuuw1z5szB9ddfDx8fH3z//fdQKpVd8kxEnUkaytp8+rIcUgorDdCoFBjWMwDj+9vDTiv7dmpNFiz5+gQmv5mEpd87L9iZU1qD3605yB4gInJ7XboR6MSJE5td/l4QBCxduhRLly5t8hytVotVq1Zh1apVnXCHRN3L9MEReOvnZOy6UICvD2eh2mhbPHNU7yBoVEq5ifl4ZinKqk3Q+zS+IzsAZJfW4PHPD+OEfcir/iyvdUeysP1cPkRRxA32kEVE5I66bc8OETXUL1yHZ6b0BwD85dvT+OaIbWsUaYXlSL03+oX5wSoCey4WNnmdw+nFmL1qD05klcFfa/tvnrTCKtQY61YeP5NbDgDILatt9BpERO6CYYfIzTw+oS9u6BeCGpNFrspc1ydYPi5VYbaeaXytqZ9O5eGeD/ejqMqIQVH+2PjkDQjx84JVBC5crtuK5UyOLew4TmsnInJHDDtEbkahEPDmnGEIsS9O6OulxJBovXx8+mDbsg0bjuXg68POm+J+sjcNj39xGAazFZMHhuHrx8YiJsgHAyJsa02dtVdzKg1mpNtnfJVWm5wqPkRE7oZhh8gNheo0ePu3w6BVKzBjSCTUyrp/la+NDcITE/sCAP7wzQn8crEQmcXVePiTg/i/705DFIF7R/fE+/eNgLeXrZF/YKSt6V8KO+fzyuHYTsfqDhG5sy5tUCai9rs+LgSHX5wCH6+GMw8XT41HZkkNvj+eg0c/Owyz1YpakxUqhYDF0+Lx6Pg+Tis5y5WdPNswljSEJcktq0FsE7u1ExF1d6zsELkxX42q0e0nFAoBr901BKN6B6LSYEatyYrr+gRh01M34LEJfRt8ZmBk3TCWKIpyc7IkrxVNyhfzK/HIp4dwKK24A09EROR6rOwQeSitWokPHxiJVdsvYki0HrOHRjW5L1ffMF+oFAIqas3IKavFmVxbhUenUaHCYG5xRpbRbMWCL4/gXF4Fcspq8P2CcS3uAbYvpQjP/vc4lt2RiAn9ObWdiDoPKztEHizAxwt/npmAW4f1aH5zXJUScWF+AIBT2WU4Z6/sjI+3hRDHyo4oijiXV45aU13T8urtyThnHwI7lV3equ0qPt+XjuzSmgZN1ERErsawQ0QAgAERtiblTSdzYTBb4eOlxHWxQQCc19r54UQupr+1Gze/vRsHUotxKrsM/9h5CQDQx97X89mv6c1+lyiKOJRm28D3Ql5Fs+cSEXUUww4RAajr2/nptG19ngEROvQI9AYA5JXXyOcdtPfkpBZWYc4/f8UD/z4Ai1XEjMRIvDFnKADg+xM5KKkyAgC2n7uMKW8mYfu5y/I1sktr5Blelwq4LxcRdS6GHSICAAywh51aky14JET5I8LfHnYcKjvSwoMJ9vOLq4wI9vXCy7cOwrCYAAzu4Q+j2Yr/Hc7E4fQSPP75ESTnV+KfSSnyNQ6nl8i/NltFpBZWde7DEdFVjWGHiADUrbUjSYjUI1KvBQAUVhphMNt6dJIvVwIAXrkzEV88PBo3D47Au/deg2A/DQRBwP3X9QIAfPxLGh765CAM9qrNofQSudpzsN6MrfOXOZRFRJ2HYYeIAABhOi1C/Lzk1wlR/gjwUUOjsv0xkV9uQFGlAUVVRggCEBfmh+vjQvDefSMw2mG7illDo6DTqpBbVovSahOGRusRF+YHi1XEzgv5ACD364TqbKtAs2+HiDoTww4RyaTFBRUCEB+ugyAIcnUnt6wWF+xVnZhAH/h4Nb5yhY+XCr8ZEQMA6BXsg4/mjcLN9i0stp3JR1mNSa7kzBkZDYCVHSLqXAw7RCSThrJiQ3zlrSQi5LBTg+R8WyjpH+7X7HWemdofL84YiK8eGYMQPw0mDwwHAOw8n48DqcUQRVsQuj4uBABwnpUdIupEDDtEJJN2TJ/QP0x+L1Jf16QshZJ+4bqGH3bgp1Hh4Rv6yEEpsYceYToNqowWrN5xEQAwolcg4u3XySiuRrXR3Or7NFusuFRQCdFxAy8ioiYw7BCRbHz/UOxYPBHPTY+X34twGMaSmpNbquzUp1AIuMle3TmeWQoAGNU7CMF+Gnn3dunarfHRnlTc9EYSvjqY2ab7IKKrE8MOETmJDfGFVl23uWikwzDWBfswVr+w5is7jZmSEOb0emSvQABAfIQtOLWlb0eazfXfQww7RNQyhh0ialaEvy3snMouR2m1CQr7TKy2Gts3BN72EKX3VqNvqO0a/e1DWW2ZkZVVYlvk8EhGKfLLW96klIiubgw7RNQsqWcnu9QWMHoG+ThVflpLq1ZifH9bQ/LIXoFQKGx7dUnbVDRW2akxWrDhaDZqjHX7cImiiMziavn1ljOXG3yOiMgRww4RNUvq2ZH0b6E5uTmPjO+DPiG+eGBs7wbXa2xG1ss/nMair47h37+kyu+VVptQ5RB+Ntu3tyAiakrjC2UQEdkF+3pBrRRgsthmPnUk7IzoFYTtiyc6vSfN7MqvMKCkyohAX9vChmXVJqw/mg3AthO7JLPEVtXxUilgNFvx66UilNWYoPdWt/u+iMizsbJDRM1SKASE+9dVd/q1cSZWS/w0KkTbNxy94DCU9c2RLHmfrpSCur2zMottw2mDo/wRF+YHs1XEjnP5Lr0nIvIsDDtE1KJIh6GsjlR2miKtt3POPpQliiK+2J8uH08tqoLFaqssZdkrOzFBPpg2yDadnUNZRNQchh0ialGEvUlZqRDQJ9TX5dcfEh0AAHh350XkldViX0oxLhVUwcdLCS+lbbgqx94gLQ1jxQT6YNog2zYUO88XoNZkafTarVVRa8L+lCIuVEjkgRh2iKhFUmWnV7APNKq2z8Rqye/G9Ub/cD9cLjfg4U8P4qM9KQCA24b3QK9gHwDApQLbooPStPPoQG8k9tAjSq9FjcmCf+1Ogdlibdf3n80txy3v7MbdH+zD9ydyXfBERNSdMOwQUYt6BtkCx6AofadcX6dV46MHRyHI1wunssux7aytB+fe0T3lSpLUtyNNO48J8oEgCLhzhG0z0de3XMD0t3djaxunov94Mhd3vLtX7gXadJJhh8jTMOwQUYtuH94DL84YiOemxbd8cjvFBPngg/tHwEtp+2NpeM8ADIrSo4998cGUQtteWI6VHQB48qZ++PPMBAT4qHExvxK///QQNrayOvOfAxl44osjqDFZMDDStuP77uRCGM3tqxARUffEsENELfK1b+wZY6/wdJaRvYPw5t1D0TvYB89OsQWrPiF1lZ2CSgMMZisUAhAVYAs7aqUCD42LRdKSSZg1NAoA5CnrzTGarXhjywUAwINjeuHb+dcj2NcLlQYzDtm3owCAi/mVeD/pEgMQkRtj2CGibmXmkCjsXDIJ4/rZVluWKzsFVfJQU6TeG2ql8x9fem81Hh3fBwCw91LL1ZkfT+aisNKAMJ0GL85MgJdKgQnxtl3fd5y3DaNZrSIe//wwXtl0Dv85kOG6hySiK4phh4i6tb72np288lp5HZ4e9iGs+hIi/RHip0G10eJUnWnMmr1pAID7ruslB6cbB9g2K91uX7dnx/l8JOfbGqM5vZ3IfTHsEFG3FuDjhSD7qsq7kwsA2KadN0ahEOT9t5IuFDR5zeOZpTiWWQovpQJzr+0pv39Dv1AoFQIuFVQho6ga7yddko/tTy1GabWxw89DRFceww4RdXtS387u5EIAdc3JjZkYb6vO7DzfdNj5xF7VmTEkEqE6jfy+3luNkb0CAQCvbzmPg2klUCsFRAd6w2IV5YpPfcs3ncWov29z2qCUiLoPhh0i6vak6ecVtWYAaLZR+oa4ECgE2y7quWU1DY4XVBjw/YkcAMA8hw1JJdJQ1nfHbefcPrwH7rjGNr19y+mG09r3pRThn0kpKKgwNBmGANteXyezypo8TkSdh2GHiLo9qUlZEtNMZSfQ1wtDYwIAAEn26s7GE7mY8c5uTHhtB6auTILJImJYTIB8nqNJ9rAjeWR8X0xNsG1LkXTBeaVmg9mCP60/Kb8+f7nhzu0AYLJYcfcHv2LW6j3Yl1LU9IM2YfmPZ3H7u7+g0mBu82eJiGGHiNyANIwliW5hCvyE/rZZVUkXCrDpZC4W/ucITueUI72oGiXVJgDA4xP7NvrZfmF+6GGf1j4lIRxxYX4YFOWPHgHeqDFZsMc+lAYAH+5KwSWHTUov5DUedj7akyrv+/XfQ5nN3nt9+eW1+HB3Co5mlOJgavNN10TUOIYdIur2HPfjUikERDjswt4YqW9nx/l8PLn2KKwicOc10fjm8bH4YeE47P3DjfK+WvUJgoDHJva1rfUztb/83pQE501H04uqsGr7RQDAoxNsU94vXK5osLdWVkk13t6WLL/efCoPNcbW7+P1w4lc2PdARWphVfMnE1GjGHaIqNvrGeQLpUIAYFtMUPp1UxJ76BHoo0atyQqTRcSMxEisuGsIRvQKxOAeenlBwqbcf10v7FwyCQMi/OX3ptp3WN929jL+79tTmLlqDwxmK8bFheDpyf2hVAgorzXjcrnB6VovfX8GNSYLru0dhOhAb1QZLfj5XOu3tPj2WN0CiWlFDDtE7cGwQ0TdnpdKIffpxAQ1H1QA2+7sNw20hZOJ8aFYefewFgNSS67tHQS9txol1SZ88ms6KmrNiA3xxbLbE6FVK9HbvmGpY9/OtjOXsfXMZagUAv52+2DMtq/wvOFoTqu+M7WwCscdmpqvRGXnUkElzuaWd/r3EF1JDDtE5BakJuWm1tip70+3DMQ/7rkG7983Al6qjv9Rp1Iq8ODY3tCoFLglMQKf/u5a/PzMBPS0h5z+4ToAQLJD2Fm13TZ89dANsegfrsNtw3sAAJIu5LdqzR6pqiOtM9TZlR2LVcTd//wVs1fv4ZAZeRSGHSJyCyN729a/aWwGVWMCfb0wY0gktGqly+7hmSn9cf5vN+Pde0dgfP9QKByqRVLYOW9vRC6qNOBEtq0q89C4WPmcgZH+MFlEbGxhd3VRFPHdMVsF6BH7NhjZJTWdukdXTmkNCiuNMFlEvL/zUssfIHITDDtE5BYeHd8X254Zj9+OiunqW2lUfIQt7EhbWvxyqQiiCAyI0CFMV9dQfdswaSgrG8mXK/Dd8RxsPJHboLH5ZHYZUgqroFUrcN91veDjpYRVBDIcFi7878FM/Gn9SVQ1MyX9wX8fwPgVO1o1bT29qO7a645mIae04TpFRO6IYYeI3IJSISAuTAdB6FjvTWeRh7HyK2G1itht365ivH0avGT2sCgIAnAwrQRTVu7Ck/85ivlfHsH/DmU5nfetvaozeWA4/DQq9Aq2zUhLtw9lmS1WLP3+NL7Yn4HHvzgCk6VhxSerpBpJFwqQUVyNoxklLT6D4zCZySLig10prX18om6NYYeIyAV6B/vAS6lAtdGCrJIaeWuLG+y7t0si9d7yIoW+Xkp5o9Nlm86iuMrWx3MxvxJr7bus3zrM1ucTG2LrDZJ6ac7lVaDaPoV914UCPPf1CVitztWhvRfrFjA8ld1y03Ga/doJkbZZaGsPZqCw0tDcR4jcAsMOEZELqJQKeT2gjSdzkVdeC41KgVG9gxqc+497rsG+F27CyaXTsHnReAyI0KG02oRXNp1FtdGMJ744jCqjBdf1CZK3r+htr+xI1ZejmaUAgB4B3lApBKw/mo3lm846fc8vl+oWQDyd0/JWFWn2YazfXhuDodF61Jqs+Pee1Db+kyDqfhh2iIhcROrbkTYavTY2qNEGaZVSgQi9FgqFAJVSgb/fnggA+O+hLMz790FcuFyJUJ0G78wdLk+Z721fRTqt0BZIjqbbhqXuGhGNFXcNAQB8uDsVx+whSBRF/OJQ2TmT41zZOZ5ZKu8iL5GGyHoH++KJSXEAgM9+TXfaIqMzGM1WzPv4AP684VSnfg9dvRh2iIhcROrbySuvBQCM7xfa3OmyEb0CMffangCAA2nFUCoErJ473KmxOdYedqRhLKmyM7xnAO64JlpufP7qoG3468LlShRWGuCltP0xn1JYJTcp15osuO+j/Zj38UFk25uQrVYR6fbm59gQX0wZGI4Ify0qDGYcaUW/T0ccyyzFzvMF+GxfOipqTZ36XXR1YtghInIRKexIbugf0sSZDT0/PR7B9vV0Fk+Nx+g+wU7HpWGsnLIa5JXVyqFneIxtSv5v7WHpu2M5qDKY8ctF2xDW6D5B8vYa0mKBh9NLUFFrhsUq4oi9QpRbXguj2Qq1UkCkveo0uo9tCG5/ivOeXF/uz8Dn+9Jb/WwtOZFVKv/6bG7j+4sRdQTDDhGRi8Q7hJ1QncbpdUsCfLzwxe9H4525w/GofV0dRyF+XvDTqCCKdYsNxoX5Qe+jBgCMjg1C72AfVBkt2HgyVw474+JCMCjK1nB82r7uzy6H4avj9gpRuj08xQT6QGWvBl0baw87qXXDYZnF1fjj+pN4ccMpXMx3TTBxXCXasbdIFEW883MyNhzNbuxjRK3GsENE5CLRgd7wtvfo3BAX0uZp8gMi/DF7aJTTYoUSQRDQy75a87ojtr/8hzsssCgIAn4z0rYG0Zf7M7DfvkP69XEhGNRDDwA4Ze/b2X2hrnH5uL2qkir16zjsMD861lZdOppRCoPZ1rez9Uzdvl7fH29+YcTWOulQ2XHsLTqSUYI3t17Akq+Pt2rFaaKmMOwQEbmIQiEgwV5FmRDfun6dtpCCiLT/1jW9Ap2O3zUiGgrB1gNTaTAjwEeNhEj/uspOTjkKKgw447D31cnsMpgtVnlBQSlQAUDfUF+E+GlgMFtxPNNWcXEOOzkNFkNsq7JqkzwLTLpHiRTYWrPiNFFzGHaIiFzo77cPxsu3DsKsIVEuv3ZssK/T62t6OoedcH+tPFUdAMb2DYZCIchhJ/lyBbbbd1xPiPSHTqNCrcmKC5cr5TV2ejt8hyAIGC0NZaUUobTaiANptgCiVgpIKaySw0mN0YI73v0Fd//z1zbN3jqRXQoA8NOobPeYXyFviXEwta5XiENZ1BEMO0RELjQgwh8PjOnd6FBURzkOMflpVIgL82twzpyRddtpXB9na5DuEeANvbcaZquIj+zr5kyID8WQGNvw1vGsUnn9HsfvAFDXpJxajJ3nC2CxiogP12GKfWHE74/bVnr+x46LOJJRiv2pxXjn5+RWP9MJe7/OxPhQ6L3VMFlEJOdXwGIVcSi9bhbYwbQSZDpsldFW2aU1rdoygzxTtw47S5cuhSAITj8RERHycVEUsXTpUkRFRcHb2xsTJ07E6dOnu/COiYg6j7SKMgAMiwmQ1+BxNGlAGKIDveGlUmCCfasKQRAwuIetunPhciUA28rOQ6MDAABHM0rkYazewc67ykt9O4fTS/CjfShpSkI4Zg+1Va6+P56Di/mVTltL/HNXCk5mtbyIIVA3E2todIC8cvPpnHKcyytHRa0ZfhqVXF2SGrPbKvlyBSa9thOPfnaoXZ8n99etww4ADBo0CLm5ufLPyZMn5WMrVqzAm2++idWrV+PgwYOIiIjAlClTUFHBqYtE5Hkch5iu6RnQ6DlqpQJfPzYWGxeOQ3RgXXAZFKWXf+2tVmJEr0B5B/mfz+bDYLZCpRDQI8Db6Xr9wvwQ6KNGjcmCLfZ+nSkJ4ZgYHwadRoWcslo89MlBGC1WTOgfihlDImGxiljy9fFW7dAuVXaGROvlfqczOeXyENY1vQJx14hoAMD6o9nt6hHadCoPRosVv1wswmX7Gkh0den2YUelUiEiIkL+CQ21/ZeKKIp466238Kc//Ql33HEHBg8ejE8++QTV1dX48ssvm72mwWBAeXm50w8RUXcX5OsFf62tt2V4vX4dRxF6LfrVm/Yu9e0AwHV9gqBRKTHMHnaK7HtyxQTVTTuXKBSC05YXYToNEnvooVUrMXWQrdKeXlQNL5UCL80ehJdnD0KQrxfO5VXgvZ2Xmn2e/Ipa5JbVQhCAQT30Do3UZXJv0OjYIEwfHAGNSoFLBVVODcytlXShbqr9jnP5bf58SypqTVi+6WynL75I7dftw05ycjKioqIQGxuL3/72t0hJsZVKU1NTkZeXh6lTp8rnajQaTJgwAXv37m32msuXL4der5d/YmJimj2fiKg7EAQBS6bF485rouV+nNZyDDs32Fd2DvfXygsOAs4zsRw5LnA4OSFc7keaPayuCfuxCX3RO8QXwX4aLJ09CICtjyerxLnPxrEyIw11xYX6wU+jkqtPZ3MrcCDVFhxG9Q6CTqvGZHuP0PpGGpUv5leiqIkNS0urjU47vm/vYNipMVoaVJde/ekc/pmUglc2nevQtanzdOuwM3r0aHz66afYvHkzPvzwQ+Tl5WHs2LEoKipCXl4eACA8PNzpM+Hh4fKxprzwwgsoKyuTfzIzMzvtGYiIXOn+Mb3xxpyh8FK17Y/v2BDbcJQgOE+LHxpTN7zVu95sL4nUMwNAbkwGgOv7BmNEr0AMiwnAExP7yu/PGhKJMX2CYbRYnZqViyoNmLpyF25dvQeZxdXyYoJD7L1DfUJ94aVSoNJglre6GBJtu7/b7bu/rzuShXKHLSUOpRVj2lu7cM+H+xsd4tpzsRBWEdDZZ3vtuVgorxnUVhcuV2DIS5vx2OeHYbHvMH82txxf7s+Qj3d0Kn5jak0WrPklFfkVHIJrr24ddm6++WbceeedSExMxOTJk7Fx40YAwCeffCKfU3/RLlEUW1zIS6PRwN/f3+mHiMiTKRUC1vy/a7Hm/12LvqF1s7iGOixMWL85WTLQvlZPvzA/jO1bV+VRKRX45vGx2DD/eqcNTwVBwOJp8QCAb45kI6WgEqIo4k/rTyE5vxLHs8pw+7t7seW07T9MpUCjViowIKJu+G1YTIB83QnxoegT6ouSahNW2QOUxSpi6fenYbGKOH+5AofTGw4jJZ23DWHNGRWDMJ0G1UZLg+0vWivpfAFMFhGbT1/GK5vOQhRFvPz9GdhzD0qrTfKQoCu983Myln5/Bq/8yMpRe3XrsFOfr68vEhMTkZycLM/Kql/Fyc/Pb1DtISIiW7CRZmhJhtmrKgDQK6Txyo5SIeCHheOwedF4aFQNd3FvzIhegbhxQBgsVhErtyVjw7Fs/HQ6D2qlgLgwPxRWGnAuzzaZRAo7gPNw26jYur4ktVKBP89IAACs2ZuG1MIqfH04E6ey63p4vjniPMQliqLcrzMpPgyT4m1rELVmKCuvrLbBekHS/QK2Heaf/uoYfk0pgkalQJB9X7OL+ZUtXrstrFZRHrrbfbGwUypHVwO3CjsGgwFnz55FZGQkYmNjERERga1bt8rHjUYjkpKSMHbs2C68SyIi9zE4Wg+pGF5/0UJHgiC0ee2gZ6f2B2Cbnv7i+lMAgEWT+2P9E2NxQz9bz5GXUoGBkXUBJ8Hh19fGOm+GOmlAGCb0D4XJIuIv357Ca5vPA6gbWtt4IscpoJzNrUB+hQHeaiVG9g7EJPuCizvO5zcbGg6lFWPcq9ux+H/Hnd4/f9kWrEbYV67ecMy2xtCj4/tgqD2wuTrs7EstQm6ZbfiqoMKAFPvij9Q23TrsLF68GElJSUhNTcX+/ftx1113oby8HA8++CAEQcCiRYuwbNkyrF+/HqdOncK8efPg4+ODe+65p6tvnYjILfhr1Xh59iA8Pbl/gwUFO2pQlB4zEiMBAFVGC4b3DMCj4/tAp1Xj3/NGYcm0eLz2myFOQ2AJ9iZlhdD49Po/zxwIpULA7uRCFFYa0SfUF6vvGY4ovRbltWanqo1U1RnTNxhatRLj+oXAS6lAelF1s6HhvZ2XYLaK2HEuX+7NMVusSLavUfTaXUNw82Db6EKkXovHJvaVF3i8VODasFN/5eh9KUVNnEnN6dZhJysrC3PnzkV8fDzuuOMOeHl5Yd++fejVqxcA4LnnnsOiRYvwxBNPYOTIkcjOzsaWLVug07V+p2Eioqvd/WN646nJ/Trl2k9P6Q+VQoBWrcAbvxkqT21XKxWYPykOt9objyXDYgJw98gYPDs1HjqtusH14sJ0uP+6XvLrv8xMgEalxG3D6xqYJUkXbMFnor0h20+jkleE/u5YDjKKqpFWWOVU5UkpqMTP9sBUZbTIlZq0omoYzFZ4q5XoHeyLN+YMxZJp8fjwgZHw8apbzbqjlZ3SaqN8P7UmCzadtLVqXB9nq3L9eolhpz1UXX0DzVm7dm2zxwVBwNKlS7F06dIrc0NERNQmcWF+9gZmBfqENtzeoj6lQsCrdw1p9pynJ/fHmZxyDIzUYaK9D+eOa3rg3Z2XsPN8AQorDSipMuJQmq1h2bFPaVJ8GHYnF+Ltn5Pxtr3ReUyfYKz53ShoVEqs2Zvm9F3HMksQH6HDeXu/Tv9wPygUAny8VJg/KU4+T2r6vtSBsPOPHRfx+pbzmDIwHKvuGY6tZy6jwmBGjwBvLJjUD79cLMK+lOJWTcQhZ9067BARkfsb3EPf8kltoPdR47+PjXF6Ly5Mh6HRehzPKsMDHx3AubxyWEUgPlyHXg69SDOHRuLfv6Qiv9wAlVKAwWzFrylF+MuG0/jjLQPxv0O2ytCwmAAcyyzFscwy3D0KOJ9n69cZENH47F2pspNTVosqgxm+mrb99frhrhS5B2nLmcv4/aeH5QrP7cN7YHjPAGhUChRWGnCpoKrRfdFa61xeOdYfycah9BL0C/PDmL7BGNMnGGEOay55GoYdIiLyCLcP74HjWWU4k2sLJlMSwvHijIFO54TptNjz/I3y653n8/G7NQfx1aFMJOdXoMZkwYAIHR4d3wePf3EExzJLAdTNxIqPaLxNIsDHCyF+XiisNOJSQSWGRAdAFEXsSylGYrRe3tW9PotVxBf70/H3H88CAO4aEY2NJ3Kxy2HV59uG94BWrcQ1PQPxa0oRfk0palfYOZNTjsX/Oy7/8wFse56tPZgJlULA6nuGY/rgyDZf1x0w7BARkUe4/ZpobDqVB51WhYU39nNaQ6gpE+PD8Nz0AXhl0zkcySgFADw0LhbD7M3R5/PKUW00y2FnQBNhB7ANZRVWFuNivi3sfHc8B0+tPYZRvQPx1SNj5NlsUuhIK6pCtbFu9tgTE/tiybR43D0qBv/v44OoNJgxJFovB5sxfYPxa0oR9qUUOfUttdb7SZdwJrccaqWAGweE4cYBYbiYX4ldFwpx/nIFlv14DpMHhjfYMsQTMOwQEZFH0Hur8dWjY1o+sZ5Hx/fB6ZxyfH88ByF+Xpg1NApatRLh/hpcLjdgf2oxMopt2140VdkBbENZ+1OL5Sblrw7aVuc/mFaCrw5lYu61PVFjtGDBf44gpaBuNphKIeCR8X2wZFo8BMG2F9kXD4/Gym0X8MgNfeTzrrNv27E/pajNfTsWq4hdybZq0ae/G40xDotDPj3FjHGv7kBGcTW+P5GD24dHN/h8WmEV1h3Nxn2je7rlcBfDDhERXdUEQcCKO4cgNsQX1/UJkqfCD40OwJYzl/G1vY8nVKdBsJ+myes4zsjKK6vFrw7TxJf/eBaTB4bjHzsuIqWgCuH+Gnz20GgE+3pBp1U32P5jaEwA1vy/a+u9p7f37diGyuLCWj/z+FhmKUqrTfDXqjCqt/Mmsj5eKjw0LhavbT6Pd3dcwq1DezitqVRpMOOBfx9ARnE1tp25jK8fHwMfr7bFh6ySakQHNr5C95XgebUqIiKiNvL2UuKZKf0xtm/dBqvSUNaWM7bp380NYQEOYaegEt8dz4Yo2tYKGhTlj/JaMx7+5KA822vFXUPRP1yHYD9Nq/c506iU8oKGe9s4BX3nedt0+hv6hzY6THX/mF7QaVVIzq+Un1ey9LvTcmXrTG45nv3vcVitrVvJubDSgD98cwLjV+xw2pD1SmPYISIiasQwe8+PyWL7iz0+vHVhJ72oGt8cti0GeMc10Vh+RyIUAuSNT++/rleDbTtaa5x95el/7LiI/PK6jUHzy2uxensytp25jBpjw41Od56v2zajMf5aNR4c0xsAsHrHRXkm2I8nc/H14SwoBOCPtwyAl1KBTafy8JbDBq+NMVms+PeeVEx6fSfWHsyEVQT2JBe2+XldhcNYREREjUjsYdtKQ1pzcEBk85tGR/hr4adRodJgxvnLFVApBMxIjESgrxceHNsbH/+ShtgQX7xwy4B239MDY3pj/ZFsJOdX4rHPD+M/j1yH1MIq/O7jg8ixbyuhVSswLi4UL84YiN4hvsivqMXJbFvQai5k/W5cLD7ak4pT2eW476P96Bemk/flenxiXzwyvi8Cfbyw5OsTeOfnZAyM0OHmxMZnb/1p/Un81z78NyjKHy/NHoSRvYPa/dwdxcoOERFRI3RaNfo5TPFuaRhLEAT0Da1b02difBgC7RuEPj99AF6aPQif/u7aNve7OPLTqPDhAyPhr1XhSEYpHv3sMH7z3q/IKatFdKA3egR4o9Zkxbazl/H7Tw+h1mSRd35P7KFHqK7pnqMgeygDgF8uFmHN3jSU1ZgwJFqPRZNt+5z9ZmQMHh4XCwBY8vUJpDWy7cbh9GL891AWBAH4222D8d2CcV0adABWdoiIiJo0LCYAFy5XQiGgVWvb9A3zk4erbhseJb+vVSvlINFRvUN88c7c4fjdmoPy8NS1sUH44P4R0HurcTqnHPM+Pojk/Eq8+tM55JcbAACT4lseOlsyLR43DbRNSU8pqERJtQlP3dQPaoc+n+dvHoBjmaU4lF6CJ744gnVPjJWbui1WEX/59jQAYM6IGNzXjinynYGVHSIioiZIa/X0DvF12rC0KVIg8tOoMHlgeKfd18T4MPxpRgKUCgF3DO+Bzx66FgE+XhAEAYN76PGafcuNj39Jw7azl22fGdB4v44jpcI29X3utT3xpxkJeP03QxET5DyLSq1UYNU9wxHk64UzueV4+Ycz8rG1BzNwOqccOq0KS6bHu/CJO4aVHSIioibMGhqFXRcKMGNIVMsnA7hpQDje3paM313fu1XhqCMeGheLudfGNDosNmlAGO67ric+35cBg9mKQB81hkYHuOy7I/XeWHn3MMz7+AC+3J+Bs7nlmJoQgQ92XQIAPDOlP0KamaZ/pQmi43avV6ny8nLo9XqUlZXB37/5BjQiIqLmWK0iBAFdvllntdGMGe/sQWphFWYPjcI7c4e7/Dve23kJr/50zum9+HAdNj457oqsxNzav78ZdsCwQ0REnin5cgXeS7qEJybGdWjz0ObkldVi65k8/HQ6D2mF1Vh1z3Bc0zOw5Q+6AMNOGzDsEBERuZ/W/v3NBmUiIiLyaAw7RERE5NEYdoiIiMijMewQERGRR2PYISIiIo/GsENEREQejWGHiIiIPBrDDhEREXk0hh0iIiLyaAw7RERE5NEYdoiIiMijMewQERGRR2PYISIiIo/GsENEREQeTdXVN9AdiKIIwLZVPBEREbkH6e9t6e/xpjDsAKioqAAAxMTEdPGdEBERUVtVVFRAr9c3eVwQW4pDVwGr1YqcnBzodDoIgtDVt+NS5eXliImJQWZmJvz9/bv6djodn9dzXU3PCvB5Pd3V9Lyd+ayiKKKiogJRUVFQKJruzGFlB4BCoUB0dHRX30an8vf39/h/oRzxeT3X1fSsAJ/X011Nz9tZz9pcRUfCBmUiIiLyaAw7RERE5NEYdjycRqPB//3f/0Gj0XT1rVwRfF7PdTU9K8Dn9XRX0/N2h2dlgzIRERF5NFZ2iIiIyKMx7BAREZFHY9ghIiIij8awQ0RERB6NYccDLF++HKNGjYJOp0NYWBhuu+02nD9/3ukcURSxdOlSREVFwdvbGxMnTsTp06e76I5da/ny5RAEAYsWLZLf87Tnzc7Oxn333Yfg4GD4+Phg2LBhOHz4sHzck57XbDbjxRdfRGxsLLy9vdGnTx+8/PLLsFqt8jnu+ry7du3CrFmzEBUVBUEQsGHDBqfjrXkug8GAhQsXIiQkBL6+vpg9ezaysrKu4FO0XnPPazKZ8PzzzyMxMRG+vr6IiorCAw88gJycHKdreMrz1vfoo49CEAS89dZbTu972vOePXsWs2fPhl6vh06nw3XXXYeMjAz5+JV6XoYdD5CUlIT58+dj37592Lp1K8xmM6ZOnYqqqir5nBUrVuDNN9/E6tWrcfDgQURERGDKlCnyvmDu6uDBg/jggw8wZMgQp/c96XlLSkpw/fXXQ61WY9OmTThz5gzeeOMNBAQEyOd40vO++uqreP/997F69WqcPXsWK1aswGuvvYZVq1bJ57jr81ZVVWHo0KFYvXp1o8db81yLFi3C+vXrsXbtWuzZsweVlZWYOXMmLBbLlXqMVmvueaurq3HkyBH8+c9/xpEjR7Bu3TpcuHABs2fPdjrPU57X0YYNG7B//35ERUU1OOZJz3vp0iWMGzcOAwYMwM6dO3H8+HH8+c9/hlarlc+5Ys8rksfJz88XAYhJSUmiKIqi1WoVIyIixFdeeUU+p7a2VtTr9eL777/fVbfZYRUVFWK/fv3ErVu3ihMmTBCfeuopURQ973mff/55cdy4cU0e97TnnTFjhvi73/3O6b077rhDvO+++0RR9JznBSCuX79eft2a5yotLRXVarW4du1a+Zzs7GxRoVCIP/300xW79/ao/7yNOXDggAhATE9PF0XRM583KytL7NGjh3jq1CmxV69e4sqVK+Vjnva8d999t/zvbWOu5POysuOBysrKAABBQUEAgNTUVOTl5WHq1KnyORqNBhMmTMDevXu75B5dYf78+ZgxYwYmT57s9L6nPe93332HkSNH4je/+Q3CwsIwfPhwfPjhh/JxT3vecePG4eeff8aFCxcAAMePH8eePXtwyy23APC855W05rkOHz4Mk8nkdE5UVBQGDx7s1s8uKSsrgyAIctXS057XarXi/vvvx5IlSzBo0KAGxz3pea1WKzZu3Ij+/ftj2rRpCAsLw+jRo52Guq7k8zLseBhRFPHMM89g3LhxGDx4MAAgLy8PABAeHu50bnh4uHzM3axduxZHjhzB8uXLGxzztOdNSUnBe++9h379+mHz5s147LHH8OSTT+LTTz8F4HnP+/zzz2Pu3LkYMGAA1Go1hg8fjkWLFmHu3LkAPO95Ja15rry8PHh5eSEwMLDJc9xVbW0t/vCHP+Cee+6RN4v0tOd99dVXoVKp8OSTTzZ63JOeNz8/H5WVlXjllVcwffp0bNmyBbfffjvuuOMOJCUlAbiyz8tdzz3MggULcOLECezZs6fBMUEQnF6LotjgPXeQmZmJp556Clu2bHEa+63PU57XarVi5MiRWLZsGQBg+PDhOH36NN577z088MAD8nme8rxfffUVPv/8c3z55ZcYNGgQjh07hkWLFiEqKgoPPvigfJ6nPG997Xkud392k8mE3/72t7BarXj33XdbPN8dn/fw4cN4++23ceTIkTbfuzs+rzSh4NZbb8XTTz8NABg2bBj27t2L999/HxMmTGjys53xvKzseJCFCxfiu+++w44dOxAdHS2/HxERAQANknJ+fn6D/4p0B4cPH0Z+fj5GjBgBlUoFlUqFpKQkvPPOO1CpVPIzecrzRkZGIiEhwem9gQMHyjMaPO33d8mSJfjDH/6A3/72t0hMTMT999+Pp59+Wq7iedrzSlrzXBERETAajSgpKWnyHHdjMpkwZ84cpKamYuvWrXJVB/Cs5929ezfy8/PRs2dP+c+t9PR0PPvss+jduzcAz3rekJAQqFSqFv/sulLPy7DjAURRxIIFC7Bu3Tps374dsbGxTsdjY2MRERGBrVu3yu8ZjUYkJSVh7NixV/p2O+ymm27CyZMncezYMfln5MiRuPfee3Hs2DH06dPHo573+uuvb7CUwIULF9CrVy8Anvf7W11dDYXC+Y8mpVIp/5eipz2vpDXPNWLECKjVaqdzcnNzcerUKbd8dinoJCcnY9u2bQgODnY67knPe//99+PEiRNOf25FRUVhyZIl2Lx5MwDPel4vLy+MGjWq2T+7rujzurTdmbrE448/Lur1enHnzp1ibm6u/FNdXS2f88orr4h6vV5ct26dePLkSXHu3LliZGSkWF5e3oV37jqOs7FE0bOe98CBA6JKpRL//ve/i8nJyeIXX3wh+vj4iJ9//rl8jic974MPPij26NFD/OGHH8TU1FRx3bp1YkhIiPjcc8/J57jr81ZUVIhHjx4Vjx49KgIQ33zzTfHo0aPy7KPWPNdjjz0mRkdHi9u2bROPHDki3njjjeLQoUNFs9ncVY/VpOae12QyibNnzxajo6PFY8eOOf3ZZTAY5Gt4yvM2pv5sLFH0rOddt26dqFarxQ8++EBMTk4WV61aJSqVSnH37t3yNa7U8zLseAAAjf58/PHH8jlWq1X8v//7PzEiIkLUaDTi+PHjxZMnT3bdTbtY/bDjac/7/fffi4MHDxY1Go04YMAA8YMPPnA67knPW15eLj711FNiz549Ra1WK/bp00f805/+5PQXoLs+744dOxr9d/XBBx8URbF1z1VTUyMuWLBADAoKEr29vcWZM2eKGRkZXfA0LWvueVNTU5v8s2vHjh3yNTzleRvTWNjxtOf96KOPxLi4OFGr1YpDhw4VN2zY4HSNK/W8giiKomtrRURERETdB3t2iIiIyKMx7BAREZFHY9ghIiIij8awQ0RERB6NYYeIiIg8GsMOEREReTSGHSIiIvJoDDtERETk0Rh2iKhL9O7dG2+99Varz9+5cycEQUBpaWmn3RMReSauoExErTJx4kQMGzasTQGlOQUFBfD19YWPj0+rzjcajSguLkZ4eDgEQXDJPbTVzp07MWnSJJSUlCAgIKBL7oGI2k7V1TdARJ5DFEVYLBaoVC3/0RIaGtqma3t5eSEiIqK9t0ZEVzEOYxFRi+bNm4ekpCS8/fbbEAQBgiAgLS1NHlravHkzRo4cCY1Gg927d+PSpUu49dZbER4eDj8/P4waNQrbtm1zumb9YSxBEPCvf/0Lt99+O3x8fNCvXz9899138vH6w1hr1qxBQEAANm/ejIEDB8LPzw/Tp09Hbm6u/Bmz2Ywnn3wSAQEBCA4OxvPPP48HH3wQt912W5PPmp6ejlmzZiEwMBC+vr4YNGgQfvzxR6SlpWHSpEkAgMDAQAiCgHnz5gGwhbwVK1agT58+8Pb2xtChQ/H11183uPeNGzdi6NCh0Gq1GD16NE6ePNni9xJRxzHsEFGL3n77bYwZMwa///3vkZubi9zcXMTExMjHn3vuOSxfvhxnz57FkCFDUFlZiVtuuQXbtm3D0aNHMW3aNMyaNQsZGRnNfs9LL72EOXPm4MSJE7jllltw7733ori4uMnzq6ur8frrr+Ozzz7Drl27kJGRgcWLF8vHX331VXzxxRf4+OOP8csvv6C8vBwbNmxo9h7mz58Pg8GAXbt24eTJk3j11Vfh5+eHmJgYfPPNNwCA8+fPIzc3F2+//TYA4MUXX8THH3+M9957D6dPn8bTTz+N++67D0lJSU7XXrJkCV5//XUcPHgQYWFhmD17NkwmU7PfS0Qu4PJ91InII02YMEF86qmnnN7bsWOHCEDcsGFDi59PSEgQV61aJb/u1auXuHLlSvk1APHFF1+UX1dWVoqCIIibNm1y+q6SkhJRFEXx448/FgGIFy9elD/zj3/8QwwPD5dfh4eHi6+99pr82mw2iz179hRvvfXWJu8zMTFRXLp0aaPH6t+DdJ9arVbcu3ev07kPPfSQOHfuXKfPrV27Vj5eVFQkent7i1999VWL30tEHcOeHSLqsJEjRzq9rqqqwksvvYQffvgBOTk5MJvNqKmpabGyM2TIEPnXvr6+0Ol0yM/Pb/J8Hx8f9O3bV34dGRkpn19WVobLly/j2muvlY8rlUqMGDECVqu1yWs++eSTePzxx7FlyxZMnjwZd955p9N91XfmzBnU1tZiypQpTu8bjUYMHz7c6b0xY8bIvw4KCkJ8fDzOnj3bru8lotbjMBYRdZivr6/T6yVLluCbb77B3//+d+zevRvHjh1DYmIijEZjs9dRq9VOrwVBaDaYNHa+WG+Caf2ZW/WP1/fwww8jJSUF999/P06ePImRI0di1apVTZ4v3d/GjRtx7Ngx+efMmTNOfTtNke6vrd9LRK3HsENEreLl5QWLxdKqc3fv3o158+bh9ttvR2JiIiIiIpCWlta5N1iPXq9HeHg4Dhw4IL9nsVhw9OjRFj8bExODxx57DOvWrcOzzz6LDz/8EIDtn4F0HUlCQgI0Gg0yMjIQFxfn9OPY1wQA+/btk39dUlKCCxcuYMCAAS1+LxF1DIexiKhVevfujf379yMtLQ1+fn4ICgpq8ty4uDisW7cOs2bNgiAI+POf/9xshaazLFy4EMuXL0dcXBwGDBiAVatWoaSkpNl1ehYtWoSbb74Z/fv3R0lJCbZv346BAwcCAHr16gVBEPDDDz/glltugbe3N3Q6HRYvXoynn34aVqsV48aNQ3l5Ofbu3Qs/Pz88+OCD8rVffvllBAcHIzw8HH/6058QEhIizwxr7nuJqGNY2SGiVlm8eDGUSiUSEhIQGhrabP/NypUrERgYiLFjx2LWrFmYNm0arrnmmit4tzbPP/885s6diwceeABjxoyBn58fpk2bBq1W2+RnLBYL5s+fj4EDB2L69OmIj4/Hu+++CwDo0aMHXnrpJfzhD39AeHg4FixYAAD461//ir/85S9Yvnw5Bg4ciGnTpuH7779HbGys07VfeeUVPPXUUxgxYgRyc3Px3XffOVWLmvpeIuoYrqBMRFcNq9WKgQMHYs6cOfjrX/96xb6XKy8TdS0OYxGRx0pPT8eWLVswYcIEGAwGrF69Gqmpqbjnnnu6+taI6AriMBYReSyFQoE1a9Zg1KhRuP7663Hy5Els27aNvTBEVxkOYxEREZFHY2WHiIiIPBrDDhEREXk0hh0iIiLyaAw7RERE5NEYdoiIiMijMewQERGRR2PYISIiIo/GsENEREQe7f8DRn7txmYdtokAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# 准备优化器和模型\n",
    "vocab_size = len(dataset.token2id)\n",
    "hidden_size = 32\n",
    "n_layers = 1\n",
    "dropout = 0\n",
    "n_tags = len(dataset.label2id)\n",
    "batch_size = 128\n",
    "epochs = 20\n",
    "learning_rate = 1e-2\n",
    "\n",
    "lstm_crf = LSTM_CRF(vocab_size, hidden_size, n_layers, dropout, n_tags)\n",
    "train(lstm_crf, batch_size, epochs, learning_rate)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "8a9138a5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "precision = 0.5911611785095321, recall = 0.40665236051502146, f1 = 0.4818477186043227\n",
      "precision = 0.5263653483992468, recall = 0.3423147581139008, f1 = 0.4148423005565863\n"
     ]
    }
   ],
   "source": [
    "evaluate(train_X, train_Y, lstm_crf, batch_size)\n",
    "evaluate(test_X, test_Y, lstm_crf, batch_size)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.17"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
